2024-12-09 23:56:34,773 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-09 23:56:34,783 main DEBUG Took 0.008313 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-09 23:56:34,784 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-09 23:56:34,784 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-09 23:56:34,785 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-09 23:56:34,786 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 23:56:34,792 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-09 23:56:34,803 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 23:56:34,804 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 23:56:34,804 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 23:56:34,805 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 23:56:34,805 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 23:56:34,806 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 23:56:34,806 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 23:56:34,807 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 23:56:34,807 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 23:56:34,807 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 23:56:34,808 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 23:56:34,808 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 23:56:34,809 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 23:56:34,809 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 23:56:34,809 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 23:56:34,809 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 23:56:34,810 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 23:56:34,810 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 23:56:34,810 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 23:56:34,811 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 23:56:34,811 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 23:56:34,811 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 23:56:34,812 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 23:56:34,812 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 23:56:34,812 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 23:56:34,813 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-09 23:56:34,814 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 23:56:34,815 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-09 23:56:34,817 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-09 23:56:34,818 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-09 23:56:34,819 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-09 23:56:34,819 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-09 23:56:34,826 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-09 23:56:34,828 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-09 23:56:34,830 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-09 23:56:34,831 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-09 23:56:34,831 main DEBUG createAppenders(={Console}) 2024-12-09 23:56:34,832 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-09 23:56:34,833 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-09 23:56:34,833 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-09 23:56:34,834 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-09 23:56:34,834 main DEBUG OutputStream closed 2024-12-09 23:56:34,834 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-09 23:56:34,835 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-09 23:56:34,835 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-09 23:56:34,902 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-09 23:56:34,904 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-09 23:56:34,905 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-09 23:56:34,906 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-09 23:56:34,906 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-09 23:56:34,907 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-09 23:56:34,907 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-09 23:56:34,907 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-09 23:56:34,907 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-09 23:56:34,908 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-09 23:56:34,908 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-09 23:56:34,908 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-09 23:56:34,908 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-09 23:56:34,909 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-09 23:56:34,909 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-09 23:56:34,909 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-09 23:56:34,909 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-09 23:56:34,910 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-09 23:56:34,912 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09 23:56:34,912 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-09 23:56:34,912 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-09 23:56:34,913 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-09T23:56:35,113 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8bc2ec3c-7335-1808-2d8f-cd0f163ec5b4 2024-12-09 23:56:35,115 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-09 23:56:35,116 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09T23:56:35,123 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-09T23:56:35,146 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-09T23:56:35,164 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8bc2ec3c-7335-1808-2d8f-cd0f163ec5b4/cluster_96f75c57-c245-065f-99c2-7563c90f684d, deleteOnExit=true 2024-12-09T23:56:35,164 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-09T23:56:35,165 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8bc2ec3c-7335-1808-2d8f-cd0f163ec5b4/test.cache.data in system properties and HBase conf 2024-12-09T23:56:35,166 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8bc2ec3c-7335-1808-2d8f-cd0f163ec5b4/hadoop.tmp.dir in system properties and HBase conf 2024-12-09T23:56:35,167 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8bc2ec3c-7335-1808-2d8f-cd0f163ec5b4/hadoop.log.dir in system properties and HBase conf 2024-12-09T23:56:35,168 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8bc2ec3c-7335-1808-2d8f-cd0f163ec5b4/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-09T23:56:35,168 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8bc2ec3c-7335-1808-2d8f-cd0f163ec5b4/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-09T23:56:35,169 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-09T23:56:35,284 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-09T23:56:35,375 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T23:56:35,379 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8bc2ec3c-7335-1808-2d8f-cd0f163ec5b4/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-09T23:56:35,379 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8bc2ec3c-7335-1808-2d8f-cd0f163ec5b4/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-09T23:56:35,380 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8bc2ec3c-7335-1808-2d8f-cd0f163ec5b4/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-09T23:56:35,380 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8bc2ec3c-7335-1808-2d8f-cd0f163ec5b4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T23:56:35,381 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8bc2ec3c-7335-1808-2d8f-cd0f163ec5b4/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-09T23:56:35,381 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8bc2ec3c-7335-1808-2d8f-cd0f163ec5b4/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-09T23:56:35,382 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8bc2ec3c-7335-1808-2d8f-cd0f163ec5b4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T23:56:35,382 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8bc2ec3c-7335-1808-2d8f-cd0f163ec5b4/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T23:56:35,383 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8bc2ec3c-7335-1808-2d8f-cd0f163ec5b4/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-09T23:56:35,383 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8bc2ec3c-7335-1808-2d8f-cd0f163ec5b4/nfs.dump.dir in system properties and HBase conf 2024-12-09T23:56:35,384 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8bc2ec3c-7335-1808-2d8f-cd0f163ec5b4/java.io.tmpdir in system properties and HBase conf 2024-12-09T23:56:35,384 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8bc2ec3c-7335-1808-2d8f-cd0f163ec5b4/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T23:56:35,385 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8bc2ec3c-7335-1808-2d8f-cd0f163ec5b4/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-09T23:56:35,385 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8bc2ec3c-7335-1808-2d8f-cd0f163ec5b4/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-09T23:56:36,331 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-09T23:56:36,391 INFO [Time-limited test {}] log.Log(170): Logging initialized @2230ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-09T23:56:36,448 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T23:56:36,498 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T23:56:36,513 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T23:56:36,514 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T23:56:36,515 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-09T23:56:36,525 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T23:56:36,527 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8bc2ec3c-7335-1808-2d8f-cd0f163ec5b4/hadoop.log.dir/,AVAILABLE} 2024-12-09T23:56:36,528 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T23:56:36,690 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8bc2ec3c-7335-1808-2d8f-cd0f163ec5b4/java.io.tmpdir/jetty-localhost-33559-hadoop-hdfs-3_4_1-tests_jar-_-any-8573190270550541799/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T23:56:36,695 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:33559} 2024-12-09T23:56:36,696 INFO [Time-limited test {}] server.Server(415): Started @2535ms 2024-12-09T23:56:37,165 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T23:56:37,173 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T23:56:37,174 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T23:56:37,174 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T23:56:37,174 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-09T23:56:37,175 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8bc2ec3c-7335-1808-2d8f-cd0f163ec5b4/hadoop.log.dir/,AVAILABLE} 2024-12-09T23:56:37,176 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T23:56:37,272 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8bc2ec3c-7335-1808-2d8f-cd0f163ec5b4/java.io.tmpdir/jetty-localhost-34039-hadoop-hdfs-3_4_1-tests_jar-_-any-11910269602526279140/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T23:56:37,273 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:34039} 2024-12-09T23:56:37,273 INFO [Time-limited test {}] server.Server(415): Started @3112ms 2024-12-09T23:56:37,317 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T23:56:38,161 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8bc2ec3c-7335-1808-2d8f-cd0f163ec5b4/cluster_96f75c57-c245-065f-99c2-7563c90f684d/data/data1/current/BP-1787334953-172.17.0.2-1733788595887/current, will proceed with Du for space computation calculation, 2024-12-09T23:56:38,161 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8bc2ec3c-7335-1808-2d8f-cd0f163ec5b4/cluster_96f75c57-c245-065f-99c2-7563c90f684d/data/data2/current/BP-1787334953-172.17.0.2-1733788595887/current, will proceed with Du for space computation calculation, 2024-12-09T23:56:38,192 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T23:56:38,230 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd0d9f765097fc7d2 with lease ID 0xf6a094ab9fcc02e9: Processing first storage report for DS-f4256d7e-a87a-457e-b173-7522b9256e00 from datanode DatanodeRegistration(127.0.0.1:42911, datanodeUuid=c0f3510f-a7b2-4204-a23a-5dac7990f2a7, infoPort=42989, infoSecurePort=0, ipcPort=45579, storageInfo=lv=-57;cid=testClusterID;nsid=365843609;c=1733788595887) 2024-12-09T23:56:38,232 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd0d9f765097fc7d2 with lease ID 0xf6a094ab9fcc02e9: from storage DS-f4256d7e-a87a-457e-b173-7522b9256e00 node DatanodeRegistration(127.0.0.1:42911, datanodeUuid=c0f3510f-a7b2-4204-a23a-5dac7990f2a7, infoPort=42989, infoSecurePort=0, ipcPort=45579, storageInfo=lv=-57;cid=testClusterID;nsid=365843609;c=1733788595887), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-09T23:56:38,232 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd0d9f765097fc7d2 with lease ID 0xf6a094ab9fcc02e9: Processing first storage report for DS-fefc7a3c-309d-4e69-b281-985350b9a1d6 from datanode DatanodeRegistration(127.0.0.1:42911, datanodeUuid=c0f3510f-a7b2-4204-a23a-5dac7990f2a7, infoPort=42989, infoSecurePort=0, ipcPort=45579, storageInfo=lv=-57;cid=testClusterID;nsid=365843609;c=1733788595887) 2024-12-09T23:56:38,232 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd0d9f765097fc7d2 with lease ID 0xf6a094ab9fcc02e9: from storage DS-fefc7a3c-309d-4e69-b281-985350b9a1d6 node DatanodeRegistration(127.0.0.1:42911, datanodeUuid=c0f3510f-a7b2-4204-a23a-5dac7990f2a7, infoPort=42989, infoSecurePort=0, ipcPort=45579, storageInfo=lv=-57;cid=testClusterID;nsid=365843609;c=1733788595887), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T23:56:38,293 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8bc2ec3c-7335-1808-2d8f-cd0f163ec5b4 2024-12-09T23:56:38,353 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8bc2ec3c-7335-1808-2d8f-cd0f163ec5b4/cluster_96f75c57-c245-065f-99c2-7563c90f684d/zookeeper_0, clientPort=60257, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8bc2ec3c-7335-1808-2d8f-cd0f163ec5b4/cluster_96f75c57-c245-065f-99c2-7563c90f684d/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8bc2ec3c-7335-1808-2d8f-cd0f163ec5b4/cluster_96f75c57-c245-065f-99c2-7563c90f684d/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-09T23:56:38,361 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60257 2024-12-09T23:56:38,370 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T23:56:38,372 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T23:56:38,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741825_1001 (size=7) 2024-12-09T23:56:38,977 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e with version=8 2024-12-09T23:56:38,977 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/hbase-staging 2024-12-09T23:56:39,049 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-09T23:56:39,260 INFO [Time-limited test {}] client.ConnectionUtils(128): master/2907d75fbb3e:0 server-side Connection retries=45 2024-12-09T23:56:39,268 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T23:56:39,269 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T23:56:39,273 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T23:56:39,273 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T23:56:39,273 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T23:56:39,387 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-09T23:56:39,441 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-09T23:56:39,452 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-09T23:56:39,457 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T23:56:39,479 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 107905 (auto-detected) 2024-12-09T23:56:39,480 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-09T23:56:39,500 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45191 2024-12-09T23:56:39,519 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45191 connecting to ZooKeeper ensemble=127.0.0.1:60257 2024-12-09T23:56:39,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:451910x0, quorum=127.0.0.1:60257, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T23:56:39,642 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45191-0x1000d103ede0000 connected 2024-12-09T23:56:39,739 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T23:56:39,743 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T23:56:39,753 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45191-0x1000d103ede0000, quorum=127.0.0.1:60257, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T23:56:39,756 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e, hbase.cluster.distributed=false 2024-12-09T23:56:39,775 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45191-0x1000d103ede0000, quorum=127.0.0.1:60257, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T23:56:39,779 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45191 2024-12-09T23:56:39,780 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45191 2024-12-09T23:56:39,780 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45191 2024-12-09T23:56:39,781 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45191 2024-12-09T23:56:39,781 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45191 2024-12-09T23:56:39,866 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2907d75fbb3e:0 server-side Connection retries=45 2024-12-09T23:56:39,867 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T23:56:39,867 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T23:56:39,867 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T23:56:39,867 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T23:56:39,868 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T23:56:39,870 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T23:56:39,872 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T23:56:39,873 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37627 2024-12-09T23:56:39,874 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37627 connecting to ZooKeeper ensemble=127.0.0.1:60257 2024-12-09T23:56:39,875 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T23:56:39,878 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T23:56:39,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:376270x0, quorum=127.0.0.1:60257, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T23:56:39,904 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:376270x0, quorum=127.0.0.1:60257, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T23:56:39,904 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37627-0x1000d103ede0001 connected 2024-12-09T23:56:39,907 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T23:56:39,915 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T23:56:39,917 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37627-0x1000d103ede0001, quorum=127.0.0.1:60257, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T23:56:39,922 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37627-0x1000d103ede0001, quorum=127.0.0.1:60257, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T23:56:39,923 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37627 2024-12-09T23:56:39,923 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37627 2024-12-09T23:56:39,924 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37627 2024-12-09T23:56:39,924 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37627 2024-12-09T23:56:39,925 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37627 2024-12-09T23:56:39,937 DEBUG [M:0;2907d75fbb3e:45191 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;2907d75fbb3e:45191 2024-12-09T23:56:39,938 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/2907d75fbb3e,45191,1733788599119 2024-12-09T23:56:39,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45191-0x1000d103ede0000, quorum=127.0.0.1:60257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T23:56:39,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37627-0x1000d103ede0001, quorum=127.0.0.1:60257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T23:56:39,977 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45191-0x1000d103ede0000, quorum=127.0.0.1:60257, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/2907d75fbb3e,45191,1733788599119 2024-12-09T23:56:40,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45191-0x1000d103ede0000, quorum=127.0.0.1:60257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T23:56:40,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37627-0x1000d103ede0001, quorum=127.0.0.1:60257, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T23:56:40,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37627-0x1000d103ede0001, quorum=127.0.0.1:60257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T23:56:40,012 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45191-0x1000d103ede0000, quorum=127.0.0.1:60257, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-09T23:56:40,013 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/2907d75fbb3e,45191,1733788599119 from backup master directory 2024-12-09T23:56:40,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45191-0x1000d103ede0000, quorum=127.0.0.1:60257, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/2907d75fbb3e,45191,1733788599119 2024-12-09T23:56:40,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37627-0x1000d103ede0001, quorum=127.0.0.1:60257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T23:56:40,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45191-0x1000d103ede0000, quorum=127.0.0.1:60257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T23:56:40,025 WARN [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T23:56:40,025 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=2907d75fbb3e,45191,1733788599119 2024-12-09T23:56:40,027 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-09T23:56:40,029 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-09T23:56:40,079 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/hbase.id] with ID: cbe338b0-2c12-4f81-bb85-08f4438f065e 2024-12-09T23:56:40,079 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/.tmp/hbase.id 2024-12-09T23:56:40,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741826_1002 (size=42) 2024-12-09T23:56:40,493 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/.tmp/hbase.id]:[hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/hbase.id] 2024-12-09T23:56:40,540 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T23:56:40,544 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-09T23:56:40,560 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 14ms. 2024-12-09T23:56:40,603 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45191-0x1000d103ede0000, quorum=127.0.0.1:60257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T23:56:40,603 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37627-0x1000d103ede0001, quorum=127.0.0.1:60257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T23:56:40,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741827_1003 (size=196) 2024-12-09T23:56:41,039 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T23:56:41,041 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-09T23:56:41,055 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:41,058 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T23:56:41,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741828_1004 (size=1189) 2024-12-09T23:56:41,510 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/MasterData/data/master/store 2024-12-09T23:56:41,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741829_1005 (size=34) 2024-12-09T23:56:41,940 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-09T23:56:41,943 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:56:41,944 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T23:56:41,945 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T23:56:41,945 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T23:56:41,946 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T23:56:41,946 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T23:56:41,946 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T23:56:41,948 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733788601944Disabling compacts and flushes for region at 1733788601944Disabling writes for close at 1733788601946 (+2 ms)Writing region close event to WAL at 1733788601946Closed at 1733788601946 2024-12-09T23:56:41,950 WARN [master/2907d75fbb3e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/MasterData/data/master/store/.initializing 2024-12-09T23:56:41,950 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/MasterData/WALs/2907d75fbb3e,45191,1733788599119 2024-12-09T23:56:41,957 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T23:56:41,970 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2907d75fbb3e%2C45191%2C1733788599119, suffix=, logDir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/MasterData/WALs/2907d75fbb3e,45191,1733788599119, archiveDir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/MasterData/oldWALs, maxLogs=10 2024-12-09T23:56:41,989 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/MasterData/WALs/2907d75fbb3e,45191,1733788599119/2907d75fbb3e%2C45191%2C1733788599119.1733788601974, exclude list is [], retry=0 2024-12-09T23:56:42,003 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42911,DS-f4256d7e-a87a-457e-b173-7522b9256e00,DISK] 2024-12-09T23:56:42,006 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-09T23:56:42,034 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/MasterData/WALs/2907d75fbb3e,45191,1733788599119/2907d75fbb3e%2C45191%2C1733788599119.1733788601974 2024-12-09T23:56:42,035 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42989:42989)] 2024-12-09T23:56:42,036 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-09T23:56:42,036 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:56:42,039 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T23:56:42,040 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T23:56:42,071 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T23:56:42,091 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-09T23:56:42,094 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:56:42,096 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T23:56:42,096 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T23:56:42,099 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-09T23:56:42,100 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:56:42,101 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:56:42,101 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T23:56:42,103 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-09T23:56:42,103 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:56:42,104 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:56:42,104 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T23:56:42,106 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-09T23:56:42,107 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:56:42,108 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:56:42,108 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T23:56:42,111 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T23:56:42,112 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T23:56:42,116 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T23:56:42,117 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T23:56:42,120 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T23:56:42,124 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T23:56:42,128 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T23:56:42,130 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66702397, jitterRate=-0.006056830286979675}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T23:56:42,134 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733788602050Initializing all the Stores at 1733788602052 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733788602053 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788602053Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788602054 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788602054Cleaning up temporary data from old regions at 1733788602117 (+63 ms)Region opened successfully at 1733788602134 (+17 ms) 2024-12-09T23:56:42,136 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-09T23:56:42,168 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b5ce06e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2907d75fbb3e/172.17.0.2:0 2024-12-09T23:56:42,198 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-09T23:56:42,206 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-09T23:56:42,207 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-09T23:56:42,209 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T23:56:42,210 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-09T23:56:42,214 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-09T23:56:42,214 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-09T23:56:42,235 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-09T23:56:42,246 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45191-0x1000d103ede0000, quorum=127.0.0.1:60257, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-09T23:56:42,315 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-09T23:56:42,321 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-09T23:56:42,324 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45191-0x1000d103ede0000, quorum=127.0.0.1:60257, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-09T23:56:42,335 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-09T23:56:42,337 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-09T23:56:42,341 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45191-0x1000d103ede0000, quorum=127.0.0.1:60257, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-09T23:56:42,352 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-09T23:56:42,355 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45191-0x1000d103ede0000, quorum=127.0.0.1:60257, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-09T23:56:42,365 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-09T23:56:42,384 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45191-0x1000d103ede0000, quorum=127.0.0.1:60257, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-09T23:56:42,393 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-09T23:56:42,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37627-0x1000d103ede0001, quorum=127.0.0.1:60257, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T23:56:42,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45191-0x1000d103ede0000, quorum=127.0.0.1:60257, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T23:56:42,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37627-0x1000d103ede0001, quorum=127.0.0.1:60257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T23:56:42,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45191-0x1000d103ede0000, quorum=127.0.0.1:60257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T23:56:42,411 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=2907d75fbb3e,45191,1733788599119, sessionid=0x1000d103ede0000, setting cluster-up flag (Was=false) 2024-12-09T23:56:42,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37627-0x1000d103ede0001, quorum=127.0.0.1:60257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T23:56:42,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45191-0x1000d103ede0000, quorum=127.0.0.1:60257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T23:56:42,477 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-09T23:56:42,481 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2907d75fbb3e,45191,1733788599119 2024-12-09T23:56:42,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37627-0x1000d103ede0001, quorum=127.0.0.1:60257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T23:56:42,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45191-0x1000d103ede0000, quorum=127.0.0.1:60257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T23:56:42,535 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-09T23:56:42,539 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2907d75fbb3e,45191,1733788599119 2024-12-09T23:56:42,549 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-09T23:56:42,608 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T23:56:42,617 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-09T23:56:42,622 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-09T23:56:42,626 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 2907d75fbb3e,45191,1733788599119 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-09T23:56:42,631 INFO [RS:0;2907d75fbb3e:37627 {}] regionserver.HRegionServer(746): ClusterId : cbe338b0-2c12-4f81-bb85-08f4438f065e 2024-12-09T23:56:42,633 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/2907d75fbb3e:0, corePoolSize=5, maxPoolSize=5 2024-12-09T23:56:42,633 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/2907d75fbb3e:0, corePoolSize=5, maxPoolSize=5 2024-12-09T23:56:42,633 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/2907d75fbb3e:0, corePoolSize=5, maxPoolSize=5 2024-12-09T23:56:42,633 DEBUG [RS:0;2907d75fbb3e:37627 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T23:56:42,633 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/2907d75fbb3e:0, corePoolSize=5, maxPoolSize=5 2024-12-09T23:56:42,633 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/2907d75fbb3e:0, corePoolSize=10, maxPoolSize=10 2024-12-09T23:56:42,633 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-09T23:56:42,634 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/2907d75fbb3e:0, corePoolSize=2, maxPoolSize=2 2024-12-09T23:56:42,634 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-09T23:56:42,635 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733788632635 2024-12-09T23:56:42,636 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-09T23:56:42,637 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-09T23:56:42,639 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T23:56:42,639 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-09T23:56:42,640 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-09T23:56:42,641 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-09T23:56:42,641 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-09T23:56:42,641 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-09T23:56:42,642 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T23:56:42,646 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:56:42,646 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-09T23:56:42,649 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-09T23:56:42,650 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-09T23:56:42,650 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-09T23:56:42,655 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-09T23:56:42,655 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-09T23:56:42,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741831_1007 (size=1321) 2024-12-09T23:56:42,657 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/2907d75fbb3e:0:becomeActiveMaster-HFileCleaner.large.0-1733788602657,5,FailOnTimeoutGroup] 2024-12-09T23:56:42,659 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/2907d75fbb3e:0:becomeActiveMaster-HFileCleaner.small.0-1733788602658,5,FailOnTimeoutGroup] 2024-12-09T23:56:42,659 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T23:56:42,659 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-09T23:56:42,660 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-09T23:56:42,661 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-09T23:56:42,662 DEBUG [RS:0;2907d75fbb3e:37627 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T23:56:42,662 DEBUG [RS:0;2907d75fbb3e:37627 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T23:56:42,675 DEBUG [RS:0;2907d75fbb3e:37627 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T23:56:42,675 DEBUG [RS:0;2907d75fbb3e:37627 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78de543e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2907d75fbb3e/172.17.0.2:0 2024-12-09T23:56:42,688 DEBUG [RS:0;2907d75fbb3e:37627 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;2907d75fbb3e:37627 2024-12-09T23:56:42,690 INFO [RS:0;2907d75fbb3e:37627 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T23:56:42,691 INFO [RS:0;2907d75fbb3e:37627 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T23:56:42,691 DEBUG [RS:0;2907d75fbb3e:37627 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T23:56:42,693 INFO [RS:0;2907d75fbb3e:37627 {}] regionserver.HRegionServer(2659): reportForDuty to master=2907d75fbb3e,45191,1733788599119 with port=37627, startcode=1733788599837 2024-12-09T23:56:42,703 DEBUG [RS:0;2907d75fbb3e:37627 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T23:56:42,729 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39493, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T23:56:42,734 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45191 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:42,735 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45191 {}] master.ServerManager(517): Registering regionserver=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:42,747 DEBUG [RS:0;2907d75fbb3e:37627 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e 2024-12-09T23:56:42,747 DEBUG [RS:0;2907d75fbb3e:37627 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35525 2024-12-09T23:56:42,748 DEBUG [RS:0;2907d75fbb3e:37627 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T23:56:42,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45191-0x1000d103ede0000, quorum=127.0.0.1:60257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T23:56:42,778 DEBUG [RS:0;2907d75fbb3e:37627 {}] zookeeper.ZKUtil(111): regionserver:37627-0x1000d103ede0001, quorum=127.0.0.1:60257, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:42,778 WARN [RS:0;2907d75fbb3e:37627 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T23:56:42,778 INFO [RS:0;2907d75fbb3e:37627 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T23:56:42,778 DEBUG [RS:0;2907d75fbb3e:37627 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/WALs/2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:42,781 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2907d75fbb3e,37627,1733788599837] 2024-12-09T23:56:42,807 INFO [RS:0;2907d75fbb3e:37627 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T23:56:42,818 INFO [RS:0;2907d75fbb3e:37627 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T23:56:42,822 INFO [RS:0;2907d75fbb3e:37627 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T23:56:42,822 INFO [RS:0;2907d75fbb3e:37627 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T23:56:42,823 INFO [RS:0;2907d75fbb3e:37627 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T23:56:42,828 INFO [RS:0;2907d75fbb3e:37627 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T23:56:42,829 INFO [RS:0;2907d75fbb3e:37627 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T23:56:42,829 DEBUG [RS:0;2907d75fbb3e:37627 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-09T23:56:42,829 DEBUG [RS:0;2907d75fbb3e:37627 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-09T23:56:42,829 DEBUG [RS:0;2907d75fbb3e:37627 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-09T23:56:42,830 DEBUG [RS:0;2907d75fbb3e:37627 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-09T23:56:42,830 DEBUG [RS:0;2907d75fbb3e:37627 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-09T23:56:42,830 DEBUG [RS:0;2907d75fbb3e:37627 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2907d75fbb3e:0, corePoolSize=2, maxPoolSize=2 2024-12-09T23:56:42,830 DEBUG [RS:0;2907d75fbb3e:37627 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-09T23:56:42,830 DEBUG [RS:0;2907d75fbb3e:37627 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-09T23:56:42,830 DEBUG [RS:0;2907d75fbb3e:37627 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-09T23:56:42,830 DEBUG [RS:0;2907d75fbb3e:37627 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-09T23:56:42,830 DEBUG [RS:0;2907d75fbb3e:37627 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-09T23:56:42,831 DEBUG [RS:0;2907d75fbb3e:37627 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2907d75fbb3e:0, corePoolSize=1, maxPoolSize=1 2024-12-09T23:56:42,831 DEBUG [RS:0;2907d75fbb3e:37627 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2907d75fbb3e:0, corePoolSize=3, maxPoolSize=3 2024-12-09T23:56:42,831 DEBUG [RS:0;2907d75fbb3e:37627 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0, corePoolSize=3, maxPoolSize=3 2024-12-09T23:56:42,833 INFO [RS:0;2907d75fbb3e:37627 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T23:56:42,834 INFO [RS:0;2907d75fbb3e:37627 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T23:56:42,834 INFO [RS:0;2907d75fbb3e:37627 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T23:56:42,834 INFO [RS:0;2907d75fbb3e:37627 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T23:56:42,834 INFO [RS:0;2907d75fbb3e:37627 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T23:56:42,834 INFO [RS:0;2907d75fbb3e:37627 {}] hbase.ChoreService(168): Chore ScheduledChore name=2907d75fbb3e,37627,1733788599837-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T23:56:42,850 INFO [RS:0;2907d75fbb3e:37627 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T23:56:42,851 INFO [RS:0;2907d75fbb3e:37627 {}] hbase.ChoreService(168): Chore ScheduledChore name=2907d75fbb3e,37627,1733788599837-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T23:56:42,852 INFO [RS:0;2907d75fbb3e:37627 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T23:56:42,852 INFO [RS:0;2907d75fbb3e:37627 {}] regionserver.Replication(171): 2907d75fbb3e,37627,1733788599837 started 2024-12-09T23:56:42,866 INFO [RS:0;2907d75fbb3e:37627 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T23:56:42,867 INFO [RS:0;2907d75fbb3e:37627 {}] regionserver.HRegionServer(1482): Serving as 2907d75fbb3e,37627,1733788599837, RpcServer on 2907d75fbb3e/172.17.0.2:37627, sessionid=0x1000d103ede0001 2024-12-09T23:56:42,867 DEBUG [RS:0;2907d75fbb3e:37627 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T23:56:42,867 DEBUG [RS:0;2907d75fbb3e:37627 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:42,868 DEBUG [RS:0;2907d75fbb3e:37627 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2907d75fbb3e,37627,1733788599837' 2024-12-09T23:56:42,868 DEBUG [RS:0;2907d75fbb3e:37627 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T23:56:42,869 DEBUG [RS:0;2907d75fbb3e:37627 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T23:56:42,869 DEBUG [RS:0;2907d75fbb3e:37627 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T23:56:42,869 DEBUG [RS:0;2907d75fbb3e:37627 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T23:56:42,870 DEBUG [RS:0;2907d75fbb3e:37627 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:42,870 DEBUG [RS:0;2907d75fbb3e:37627 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2907d75fbb3e,37627,1733788599837' 2024-12-09T23:56:42,870 DEBUG [RS:0;2907d75fbb3e:37627 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T23:56:42,870 DEBUG [RS:0;2907d75fbb3e:37627 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T23:56:42,871 DEBUG [RS:0;2907d75fbb3e:37627 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T23:56:42,871 INFO [RS:0;2907d75fbb3e:37627 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T23:56:42,871 INFO [RS:0;2907d75fbb3e:37627 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T23:56:42,981 INFO [RS:0;2907d75fbb3e:37627 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T23:56:42,986 INFO [RS:0;2907d75fbb3e:37627 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2907d75fbb3e%2C37627%2C1733788599837, suffix=, logDir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/WALs/2907d75fbb3e,37627,1733788599837, archiveDir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/oldWALs, maxLogs=32 2024-12-09T23:56:43,000 DEBUG [RS:0;2907d75fbb3e:37627 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/WALs/2907d75fbb3e,37627,1733788599837/2907d75fbb3e%2C37627%2C1733788599837.1733788602988, exclude list is [], retry=0 2024-12-09T23:56:43,004 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42911,DS-f4256d7e-a87a-457e-b173-7522b9256e00,DISK] 2024-12-09T23:56:43,007 INFO [RS:0;2907d75fbb3e:37627 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/WALs/2907d75fbb3e,37627,1733788599837/2907d75fbb3e%2C37627%2C1733788599837.1733788602988 2024-12-09T23:56:43,007 DEBUG [RS:0;2907d75fbb3e:37627 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42989:42989)] 2024-12-09T23:56:43,061 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-09T23:56:43,061 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e 2024-12-09T23:56:43,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741833_1009 (size=32) 2024-12-09T23:56:43,479 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:56:43,484 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T23:56:43,486 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T23:56:43,486 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:56:43,487 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T23:56:43,487 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T23:56:43,489 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T23:56:43,489 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:56:43,490 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T23:56:43,490 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T23:56:43,492 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T23:56:43,492 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:56:43,493 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T23:56:43,493 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T23:56:43,495 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T23:56:43,495 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:56:43,496 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T23:56:43,496 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T23:56:43,497 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/hbase/meta/1588230740 2024-12-09T23:56:43,498 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/hbase/meta/1588230740 2024-12-09T23:56:43,500 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T23:56:43,500 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T23:56:43,501 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T23:56:43,503 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T23:56:43,506 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T23:56:43,507 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61805466, jitterRate=-0.07902678847312927}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T23:56:43,509 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733788603480Initializing all the Stores at 1733788603483 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733788603483Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733788603484 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788603484Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733788603484Cleaning up temporary data from old regions at 1733788603500 (+16 ms)Region opened successfully at 1733788603509 (+9 ms) 2024-12-09T23:56:43,509 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T23:56:43,509 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T23:56:43,509 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T23:56:43,509 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T23:56:43,510 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T23:56:43,511 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T23:56:43,511 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733788603509Disabling compacts and flushes for region at 1733788603509Disabling writes for close at 1733788603510 (+1 ms)Writing region close event to WAL at 1733788603510Closed at 1733788603511 (+1 ms) 2024-12-09T23:56:43,515 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T23:56:43,515 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-09T23:56:43,520 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-09T23:56:43,526 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T23:56:43,529 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-09T23:56:43,684 DEBUG [2907d75fbb3e:45191 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-09T23:56:43,696 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:43,702 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2907d75fbb3e,37627,1733788599837, state=OPENING 2024-12-09T23:56:43,757 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-09T23:56:43,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45191-0x1000d103ede0000, quorum=127.0.0.1:60257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T23:56:43,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37627-0x1000d103ede0001, quorum=127.0.0.1:60257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T23:56:43,770 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T23:56:43,770 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T23:56:43,771 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T23:56:43,773 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2907d75fbb3e,37627,1733788599837}] 2024-12-09T23:56:43,954 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T23:56:43,957 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54807, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T23:56:43,968 INFO [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-09T23:56:43,968 INFO [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T23:56:43,969 INFO [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-09T23:56:43,972 INFO [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2907d75fbb3e%2C37627%2C1733788599837.meta, suffix=.meta, logDir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/WALs/2907d75fbb3e,37627,1733788599837, archiveDir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/oldWALs, maxLogs=32 2024-12-09T23:56:43,985 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/WALs/2907d75fbb3e,37627,1733788599837/2907d75fbb3e%2C37627%2C1733788599837.meta.1733788603974.meta, exclude list is [], retry=0 2024-12-09T23:56:43,989 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42911,DS-f4256d7e-a87a-457e-b173-7522b9256e00,DISK] 2024-12-09T23:56:43,991 INFO [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/WALs/2907d75fbb3e,37627,1733788599837/2907d75fbb3e%2C37627%2C1733788599837.meta.1733788603974.meta 2024-12-09T23:56:43,992 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42989:42989)] 2024-12-09T23:56:43,992 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-09T23:56:43,994 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-09T23:56:43,996 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-09T23:56:43,999 INFO [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-09T23:56:44,003 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-09T23:56:44,003 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:56:44,004 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-09T23:56:44,004 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-09T23:56:44,006 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T23:56:44,008 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T23:56:44,008 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:56:44,009 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T23:56:44,009 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T23:56:44,010 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T23:56:44,010 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:56:44,011 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T23:56:44,011 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T23:56:44,012 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T23:56:44,012 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:56:44,013 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T23:56:44,013 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T23:56:44,014 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T23:56:44,014 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:56:44,015 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T23:56:44,015 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T23:56:44,017 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/hbase/meta/1588230740 2024-12-09T23:56:44,019 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/hbase/meta/1588230740 2024-12-09T23:56:44,021 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T23:56:44,021 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T23:56:44,022 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T23:56:44,025 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T23:56:44,026 INFO [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62718749, jitterRate=-0.0654178112745285}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T23:56:44,026 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-09T23:56:44,028 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733788604004Writing region info on filesystem at 1733788604004Initializing all the Stores at 1733788604006 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733788604006Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733788604006Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788604006Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733788604006Cleaning up temporary data from old regions at 1733788604021 (+15 ms)Running coprocessor post-open hooks at 1733788604026 (+5 ms)Region opened successfully at 1733788604028 (+2 ms) 2024-12-09T23:56:44,034 INFO [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733788603948 2024-12-09T23:56:44,043 DEBUG [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-09T23:56:44,043 INFO [RS_OPEN_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-09T23:56:44,045 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:44,046 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2907d75fbb3e,37627,1733788599837, state=OPEN 2024-12-09T23:56:44,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37627-0x1000d103ede0001, quorum=127.0.0.1:60257, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T23:56:44,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45191-0x1000d103ede0000, quorum=127.0.0.1:60257, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T23:56:44,109 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T23:56:44,109 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T23:56:44,110 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:44,116 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-09T23:56:44,117 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2907d75fbb3e,37627,1733788599837 in 337 msec 2024-12-09T23:56:44,126 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-09T23:56:44,126 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 600 msec 2024-12-09T23:56:44,127 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T23:56:44,127 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-09T23:56:44,143 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:56:44,145 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:56:44,161 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:56:44,164 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55237, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:56:44,181 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6060 sec 2024-12-09T23:56:44,181 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733788604181, completionTime=-1 2024-12-09T23:56:44,184 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-09T23:56:44,184 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T23:56:44,206 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-09T23:56:44,206 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733788664206 2024-12-09T23:56:44,206 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733788724206 2024-12-09T23:56:44,206 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 21 msec 2024-12-09T23:56:44,208 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2907d75fbb3e,45191,1733788599119-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T23:56:44,209 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2907d75fbb3e,45191,1733788599119-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T23:56:44,209 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2907d75fbb3e,45191,1733788599119-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T23:56:44,210 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-2907d75fbb3e:45191, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T23:56:44,211 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-09T23:56:44,212 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-09T23:56:44,218 DEBUG [master/2907d75fbb3e:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-09T23:56:44,236 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.211sec 2024-12-09T23:56:44,237 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-09T23:56:44,238 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-09T23:56:44,239 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-09T23:56:44,240 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-09T23:56:44,240 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-09T23:56:44,241 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2907d75fbb3e,45191,1733788599119-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T23:56:44,241 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2907d75fbb3e,45191,1733788599119-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-09T23:56:44,249 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-09T23:56:44,250 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-09T23:56:44,250 INFO [master/2907d75fbb3e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2907d75fbb3e,45191,1733788599119-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T23:56:44,346 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:56:44,348 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-09T23:56:44,348 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-09T23:56:44,350 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:56:44,352 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:56:44,359 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:56:44,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:56:44,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:56:44,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5670729d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:56:44,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:56:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:56:44,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:56:44,366 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39540, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:56:44,369 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:56:44,370 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:56:44,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:56:44,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:56:44,380 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49586, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:56:44,397 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=2907d75fbb3e,45191,1733788599119 2024-12-09T23:56:44,409 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=305, ProcessCount=11, AvailableMemoryMB=4686 2024-12-09T23:56:44,417 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T23:56:44,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 2907d75fbb3e,45191,1733788599119 2024-12-09T23:56:44,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@79901794 2024-12-09T23:56:44,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T23:56:44,429 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39542, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T23:56:44,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T23:56:44,450 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T23:56:44,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T23:56:44,462 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T23:56:44,465 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-09T23:56:44,465 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:56:44,468 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T23:56:44,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T23:56:44,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741835_1011 (size=1061) 2024-12-09T23:56:44,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T23:56:44,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T23:56:44,892 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e 2024-12-09T23:56:44,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741836_1012 (size=53) 2024-12-09T23:56:45,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T23:56:45,305 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:56:45,306 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c1cc0773bc61ee89c3c2791bfea1e203, disabling compactions & flushes 2024-12-09T23:56:45,306 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:45,306 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:45,306 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. after waiting 0 ms 2024-12-09T23:56:45,307 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:45,307 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:45,307 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c1cc0773bc61ee89c3c2791bfea1e203: Waiting for close lock at 1733788605306Disabling compacts and flushes for region at 1733788605306Disabling writes for close at 1733788605307 (+1 ms)Writing region close event to WAL at 1733788605307Closed at 1733788605307 2024-12-09T23:56:45,312 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T23:56:45,317 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733788605312"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733788605312"}]},"ts":"1733788605312"} 2024-12-09T23:56:45,321 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T23:56:45,323 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T23:56:45,325 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733788605323"}]},"ts":"1733788605323"} 2024-12-09T23:56:45,329 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T23:56:45,330 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1cc0773bc61ee89c3c2791bfea1e203, ASSIGN}] 2024-12-09T23:56:45,333 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1cc0773bc61ee89c3c2791bfea1e203, ASSIGN 2024-12-09T23:56:45,334 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1cc0773bc61ee89c3c2791bfea1e203, ASSIGN; state=OFFLINE, location=2907d75fbb3e,37627,1733788599837; forceNewPlan=false, retain=false 2024-12-09T23:56:45,487 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c1cc0773bc61ee89c3c2791bfea1e203, regionState=OPENING, regionLocation=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:45,495 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1cc0773bc61ee89c3c2791bfea1e203, ASSIGN because future has completed 2024-12-09T23:56:45,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837}] 2024-12-09T23:56:45,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T23:56:45,661 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:45,662 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} 2024-12-09T23:56:45,662 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:56:45,662 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:56:45,663 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:56:45,663 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:56:45,665 INFO [StoreOpener-c1cc0773bc61ee89c3c2791bfea1e203-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:56:45,668 INFO [StoreOpener-c1cc0773bc61ee89c3c2791bfea1e203-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T23:56:45,668 INFO [StoreOpener-c1cc0773bc61ee89c3c2791bfea1e203-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1cc0773bc61ee89c3c2791bfea1e203 columnFamilyName A 2024-12-09T23:56:45,669 DEBUG [StoreOpener-c1cc0773bc61ee89c3c2791bfea1e203-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:56:45,670 INFO [StoreOpener-c1cc0773bc61ee89c3c2791bfea1e203-1 {}] regionserver.HStore(327): Store=c1cc0773bc61ee89c3c2791bfea1e203/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:56:45,670 INFO [StoreOpener-c1cc0773bc61ee89c3c2791bfea1e203-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:56:45,672 INFO [StoreOpener-c1cc0773bc61ee89c3c2791bfea1e203-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T23:56:45,672 INFO [StoreOpener-c1cc0773bc61ee89c3c2791bfea1e203-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1cc0773bc61ee89c3c2791bfea1e203 columnFamilyName B 2024-12-09T23:56:45,672 DEBUG [StoreOpener-c1cc0773bc61ee89c3c2791bfea1e203-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:56:45,673 INFO [StoreOpener-c1cc0773bc61ee89c3c2791bfea1e203-1 {}] regionserver.HStore(327): Store=c1cc0773bc61ee89c3c2791bfea1e203/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:56:45,674 INFO [StoreOpener-c1cc0773bc61ee89c3c2791bfea1e203-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:56:45,676 INFO [StoreOpener-c1cc0773bc61ee89c3c2791bfea1e203-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T23:56:45,676 INFO [StoreOpener-c1cc0773bc61ee89c3c2791bfea1e203-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1cc0773bc61ee89c3c2791bfea1e203 columnFamilyName C 2024-12-09T23:56:45,676 DEBUG [StoreOpener-c1cc0773bc61ee89c3c2791bfea1e203-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:56:45,677 INFO [StoreOpener-c1cc0773bc61ee89c3c2791bfea1e203-1 {}] regionserver.HStore(327): Store=c1cc0773bc61ee89c3c2791bfea1e203/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:56:45,678 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:45,678 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:56:45,679 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:56:45,680 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:56:45,682 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:56:45,682 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:56:45,683 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T23:56:45,685 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:56:45,689 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T23:56:45,690 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened c1cc0773bc61ee89c3c2791bfea1e203; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67955668, jitterRate=0.01261836290359497}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T23:56:45,690 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:56:45,692 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for c1cc0773bc61ee89c3c2791bfea1e203: Running coprocessor pre-open hook at 1733788605663Writing region info on filesystem at 1733788605663Initializing all the Stores at 1733788605665 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788605665Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788605665Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788605665Cleaning up temporary data from old regions at 1733788605682 (+17 ms)Running coprocessor post-open hooks at 1733788605690 (+8 ms)Region opened successfully at 1733788605692 (+2 ms) 2024-12-09T23:56:45,694 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., pid=6, masterSystemTime=1733788605651 2024-12-09T23:56:45,697 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:45,698 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:45,699 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c1cc0773bc61ee89c3c2791bfea1e203, regionState=OPEN, openSeqNum=2, regionLocation=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:45,703 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 because future has completed 2024-12-09T23:56:45,711 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-09T23:56:45,711 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 in 209 msec 2024-12-09T23:56:45,716 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-09T23:56:45,716 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1cc0773bc61ee89c3c2791bfea1e203, ASSIGN in 381 msec 2024-12-09T23:56:45,718 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T23:56:45,718 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733788605718"}]},"ts":"1733788605718"} 2024-12-09T23:56:45,722 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T23:56:45,724 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T23:56:45,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2700 sec 2024-12-09T23:56:46,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T23:56:46,626 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:56:46,634 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:56:46,634 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:56:46,634 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:56:46,636 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:56:46,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:56:46,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:56:46,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c34a958, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:56:46,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:56:46,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:56:46,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:56:46,640 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39556, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:56:46,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@127234f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:56:46,642 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:56:46,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:56:46,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:56:46,647 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49600, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:56:46,650 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T23:56:46,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 2907d75fbb3e,45191,1733788599119 2024-12-09T23:56:46,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@42827a35 2024-12-09T23:56:46,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T23:56:46,656 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39564, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T23:56:46,660 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:56:46,660 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:56:46,660 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:56:46,661 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:56:46,662 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:56:46,662 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:56:46,662 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@89c0433, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:56:46,662 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:56:46,663 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:56:46,663 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:56:46,665 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39586, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:56:46,666 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ef39c97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:56:46,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:56:46,667 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:56:46,667 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:56:46,672 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:56:46,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:56:46,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:56:46,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@395f1c25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:56:46,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:56:46,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:56:46,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:56:46,675 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39602, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:56:46,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a7f3cf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:56:46,677 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:56:46,677 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:56:46,677 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:56:46,678 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:56:46,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:56:46,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:56:46,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ab1aa4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:56:46,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:56:46,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:56:46,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:56:46,681 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39620, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:56:46,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50c5f854, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:56:46,683 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:56:46,683 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:56:46,683 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:56:46,685 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:56:46,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:56:46,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:56:46,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51268a5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:56:46,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:56:46,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:56:46,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:56:46,688 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39650, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:56:46,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1768b9d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:56:46,689 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:56:46,689 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:56:46,690 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:56:46,691 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:56:46,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:56:46,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:56:46,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@644e9151, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:56:46,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:56:46,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:56:46,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:56:46,693 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39660, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:56:46,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ccd238a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:56:46,695 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:56:46,695 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:56:46,695 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:56:46,696 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:56:46,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:56:46,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:56:46,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@294411fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:56:46,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:56:46,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:56:46,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:56:46,699 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39666, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:56:46,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24bb446c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:56:46,701 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:56:46,701 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:56:46,701 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:56:46,702 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:56:46,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:56:46,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:56:46,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@796fbe9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:56:46,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:56:46,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:56:46,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:56:46,704 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39688, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:56:46,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74d388d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:56:46,706 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:56:46,706 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:56:46,706 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:56:46,707 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:56:46,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:56:46,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:56:46,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c24d9c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:56:46,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:56:46,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:56:46,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:56:46,710 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39710, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:56:46,710 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@276c4225, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:56:46,711 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:56:46,711 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:56:46,712 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:56:46,712 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:56:46,713 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:56:46,713 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:56:46,713 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2de480bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:56:46,713 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:56:46,713 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:56:46,714 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:56:46,715 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39728, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:56:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bc72606, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:56:46,719 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:56:46,719 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:56:46,719 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:56:46,719 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:56:46,720 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:56:46,721 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:56:46,721 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:56:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:56:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:56:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:56:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:56:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:56:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:56:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:56:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:56:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:56:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:56:46,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:56:46,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:56:46,724 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:56:46,724 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49602, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:56:46,724 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:56:46,725 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49630, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:56:46,725 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49632, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:56:46,725 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49604, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:56:46,726 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49638, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:56:46,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:56:46,726 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:56:46,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:56:46,727 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:56:46,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:56:46,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:56:46,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:56:46,730 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49656, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:56:46,731 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49660, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:56:46,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2] 2024-12-09T23:56:46,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2] 2024-12-09T23:56:46,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2] 2024-12-09T23:56:46,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2] 2024-12-09T23:56:46,740 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49646, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:56:46,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2] 2024-12-09T23:56:46,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2] 2024-12-09T23:56:46,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2] 2024-12-09T23:56:46,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2] 2024-12-09T23:56:46,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2] 2024-12-09T23:56:46,749 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:56:46,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-09T23:56:46,759 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:56:46,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T23:56:46,762 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:56:46,764 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:56:46,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:56:46,803 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1cc0773bc61ee89c3c2791bfea1e203 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T23:56:46,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=A 2024-12-09T23:56:46,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:56:46,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=B 2024-12-09T23:56:46,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:56:46,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=C 2024-12-09T23:56:46,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:56:46,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T23:56:46,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:46,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:46,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788666897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:46,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788666889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:46,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733788606876, max timestamp=1733788606909Num uniques 859; 2024-12-09T23:56:46,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T23:56:46,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/635f28a26aea4f9bbb9cf7081136bb66 is 50, key is test_row_0/A:col10/1733788606788/Put/seqid=0 2024-12-09T23:56:46,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T23:56:46,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733788606876, max timestamp=1733788606909Num uniques 859; 2024-12-09T23:56:46,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T23:56:46,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:46,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:56:46,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:46,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:46,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:46,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733788606876, max timestamp=1733788606909Num uniques 859; 2024-12-09T23:56:46,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T23:56:46,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:46,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:46,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:46,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:46,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741837_1013 (size=12001) 2024-12-09T23:56:46,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:46,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/635f28a26aea4f9bbb9cf7081136bb66 2024-12-09T23:56:47,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:47,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788667042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:47,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788667043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:47,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:47,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:47,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788667046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:47,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:47,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:47,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788667064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:47,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:47,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:47,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788667064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:47,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:47,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/80de9e803b58411c9848a24e09f80ade is 50, key is test_row_0/B:col10/1733788606788/Put/seqid=0 2024-12-09T23:56:47,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T23:56:47,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T23:56:47,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741838_1014 (size=12001) 2024-12-09T23:56:47,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:47,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:56:47,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:47,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:47,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:47,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/80de9e803b58411c9848a24e09f80ade 2024-12-09T23:56:47,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:47,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:47,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788667163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:47,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:47,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:47,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788667163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:47,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:47,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788667164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:47,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:47,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:47,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/7d27212aede44f18adb8034aef636699 is 50, key is test_row_0/C:col10/1733788606788/Put/seqid=0 2024-12-09T23:56:47,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741839_1015 (size=12001) 2024-12-09T23:56:47,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/7d27212aede44f18adb8034aef636699 2024-12-09T23:56:47,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/635f28a26aea4f9bbb9cf7081136bb66 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/635f28a26aea4f9bbb9cf7081136bb66 2024-12-09T23:56:47,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/635f28a26aea4f9bbb9cf7081136bb66, entries=150, sequenceid=13, filesize=11.7 K 2024-12-09T23:56:47,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/80de9e803b58411c9848a24e09f80ade as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/80de9e803b58411c9848a24e09f80ade 2024-12-09T23:56:47,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/80de9e803b58411c9848a24e09f80ade, entries=150, sequenceid=13, filesize=11.7 K 2024-12-09T23:56:47,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/7d27212aede44f18adb8034aef636699 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/7d27212aede44f18adb8034aef636699 2024-12-09T23:56:47,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T23:56:47,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/7d27212aede44f18adb8034aef636699, entries=150, sequenceid=13, filesize=11.7 K 2024-12-09T23:56:47,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:47,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:56:47,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:47,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:47,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:47,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:47,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for c1cc0773bc61ee89c3c2791bfea1e203 in 486ms, sequenceid=13, compaction requested=false 2024-12-09T23:56:47,294 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-09T23:56:47,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:56:47,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733788607035, max timestamp=1733788607296Num uniques 1018; 2024-12-09T23:56:47,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T23:56:47,315 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1cc0773bc61ee89c3c2791bfea1e203 3/3 column families, dataSize=109.58 KB heapSize=312.13 KB 2024-12-09T23:56:47,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733788607035, max timestamp=1733788607296Num uniques 1018; 2024-12-09T23:56:47,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T23:56:47,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=A 2024-12-09T23:56:47,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:56:47,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:56:47,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=B 2024-12-09T23:56:47,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:56:47,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=C 2024-12-09T23:56:47,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T23:56:47,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/3b31be394fd84f488e8f169c06213427 is 50, key is test_row_0/A:col10/1733788607035/Put/seqid=0 2024-12-09T23:56:47,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741840_1016 (size=16681) 2024-12-09T23:56:47,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T23:56:47,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:47,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788667411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:47,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:47,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:47,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788667413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:47,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:47,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:47,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788667415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:47,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:47,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:47,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788667418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:47,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:47,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:47,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788667422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:47,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:47,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T23:56:47,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:47,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:56:47,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:47,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:47,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:47,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:47,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:47,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788667535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:47,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:47,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788667535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:47,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:47,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:47,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:47,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788667545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:47,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:47,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788667545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:47,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:47,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:47,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788667545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:47,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:47,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:47,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T23:56:47,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:47,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:56:47,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:47,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:47,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:47,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:47,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788667754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788667754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:47,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:47,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:47,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T23:56:47,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:47,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:47,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:56:47,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788667754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:47,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:47,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:47,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:47,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:47,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:47,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788667764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788667764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:47,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:47,769 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:47,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:47,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/3b31be394fd84f488e8f169c06213427 2024-12-09T23:56:47,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/01b88dc5dc924475ab73054189622e83 is 50, key is test_row_0/B:col10/1733788607035/Put/seqid=0 2024-12-09T23:56:47,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741841_1017 (size=12001) 2024-12-09T23:56:47,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/01b88dc5dc924475ab73054189622e83 2024-12-09T23:56:47,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/34317ca0c5624706958db763e910f8a3 is 50, key is test_row_0/C:col10/1733788607035/Put/seqid=0 2024-12-09T23:56:47,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741842_1018 (size=12001) 2024-12-09T23:56:47,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/34317ca0c5624706958db763e910f8a3 2024-12-09T23:56:47,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/3b31be394fd84f488e8f169c06213427 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/3b31be394fd84f488e8f169c06213427 2024-12-09T23:56:47,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T23:56:47,916 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/3b31be394fd84f488e8f169c06213427, entries=250, sequenceid=52, filesize=16.3 K 2024-12-09T23:56:47,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/01b88dc5dc924475ab73054189622e83 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/01b88dc5dc924475ab73054189622e83 2024-12-09T23:56:47,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T23:56:47,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:47,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:56:47,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:47,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:47,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:47,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:47,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/01b88dc5dc924475ab73054189622e83, entries=150, sequenceid=52, filesize=11.7 K 2024-12-09T23:56:47,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/34317ca0c5624706958db763e910f8a3 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/34317ca0c5624706958db763e910f8a3 2024-12-09T23:56:47,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/34317ca0c5624706958db763e910f8a3, entries=150, sequenceid=52, filesize=11.7 K 2024-12-09T23:56:47,961 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~71.56 KB/73280, heapSize ~198.67 KB/203440, currentSize=107.34 KB/109920 for c1cc0773bc61ee89c3c2791bfea1e203 in 646ms, sequenceid=52, compaction requested=false 2024-12-09T23:56:47,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:56:48,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T23:56:48,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:48,080 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing c1cc0773bc61ee89c3c2791bfea1e203 3/3 column families, dataSize=107.34 KB heapSize=319.73 KB 2024-12-09T23:56:48,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:56:48,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733788607410, max timestamp=1733788608077Num uniques 859; 2024-12-09T23:56:48,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:56:48,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T23:56:48,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733788607410, max timestamp=1733788608076Num uniques 859; 2024-12-09T23:56:48,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T23:56:48,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=A 2024-12-09T23:56:48,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:56:48,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=B 2024-12-09T23:56:48,086 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733788607410, max timestamp=1733788608076Num uniques 859; 2024-12-09T23:56:48,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:56:48,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T23:56:48,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=C 2024-12-09T23:56:48,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:56:48,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/e201be93a15a4e849d4f510a182bf4f4 is 50, key is test_row_0/A:col10/1733788607414/Put/seqid=0 2024-12-09T23:56:48,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741843_1019 (size=12001) 2024-12-09T23:56:48,125 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/e201be93a15a4e849d4f510a182bf4f4 2024-12-09T23:56:48,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/6e72ff982971413bbc64347fe85f54e1 is 50, key is test_row_0/B:col10/1733788607414/Put/seqid=0 2024-12-09T23:56:48,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733788608106, max timestamp=1733788608148Num uniques 859; 2024-12-09T23:56:48,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T23:56:48,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733788608106, max timestamp=1733788608148Num uniques 859; 2024-12-09T23:56:48,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T23:56:48,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733788608106, max timestamp=1733788608146Num uniques 859; 2024-12-09T23:56:48,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T23:56:48,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741844_1020 (size=12001) 2024-12-09T23:56:48,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733788608180, max timestamp=1733788608201Num uniques 1018; 2024-12-09T23:56:48,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-12-09T23:56:48,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733788608180, max timestamp=1733788608201Num uniques 1018; 2024-12-09T23:56:48,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-12-09T23:56:48,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733788608180, max timestamp=1733788608201Num uniques 1018; 2024-12-09T23:56:48,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T23:56:48,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788668244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:48,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788668251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:48,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733788608240, max timestamp=1733788608255Num uniques 1177; 2024-12-09T23:56:48,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=8102, suffix heap size=150376,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=177 2024-12-09T23:56:48,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:48,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733788608240, max timestamp=1733788608255Num uniques 1177; 2024-12-09T23:56:48,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=8102, suffix heap size=150376,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=177 2024-12-09T23:56:48,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733788608240, max timestamp=1733788608255Num uniques 1027; 2024-12-09T23:56:48,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-09T23:56:48,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733788608311, max timestamp=1733788608325Num uniques 1036; 2024-12-09T23:56:48,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733788608311, max timestamp=1733788608325Num uniques 1036; 2024-12-09T23:56:48,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-12-09T23:56:48,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-12-09T23:56:48,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733788608311, max timestamp=1733788608325Num uniques 1036; 2024-12-09T23:56:48,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-12-09T23:56:48,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:48,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788668374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:48,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:48,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:48,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788668377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:48,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:48,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788668377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:48,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:48,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:48,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788668380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:48,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:48,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:48,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:48,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788668386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:48,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:48,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:48,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788668493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:48,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:48,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:48,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788668494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:48,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788668494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:48,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:48,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:48,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:48,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788668494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:48,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788668504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:48,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:48,565 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/6e72ff982971413bbc64347fe85f54e1 2024-12-09T23:56:48,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/9c07739a9df14ea48f85c5034a9aac56 is 50, key is test_row_0/C:col10/1733788607414/Put/seqid=0 2024-12-09T23:56:48,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741845_1021 (size=12001) 2024-12-09T23:56:48,601 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/9c07739a9df14ea48f85c5034a9aac56 2024-12-09T23:56:48,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/e201be93a15a4e849d4f510a182bf4f4 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/e201be93a15a4e849d4f510a182bf4f4 2024-12-09T23:56:48,626 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/e201be93a15a4e849d4f510a182bf4f4, entries=150, sequenceid=74, filesize=11.7 K 2024-12-09T23:56:48,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/6e72ff982971413bbc64347fe85f54e1 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/6e72ff982971413bbc64347fe85f54e1 2024-12-09T23:56:48,643 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/6e72ff982971413bbc64347fe85f54e1, entries=150, sequenceid=74, filesize=11.7 K 2024-12-09T23:56:48,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/9c07739a9df14ea48f85c5034a9aac56 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/9c07739a9df14ea48f85c5034a9aac56 2024-12-09T23:56:48,665 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/9c07739a9df14ea48f85c5034a9aac56, entries=150, sequenceid=74, filesize=11.7 K 2024-12-09T23:56:48,667 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~101.34 KB/103776, currentSize=147.60 KB/151140 for c1cc0773bc61ee89c3c2791bfea1e203 in 587ms, sequenceid=74, compaction requested=true 2024-12-09T23:56:48,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:56:48,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:48,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-09T23:56:48,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-09T23:56:48,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-09T23:56:48,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9060 sec 2024-12-09T23:56:48,682 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.9250 sec 2024-12-09T23:56:48,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:56:48,718 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1cc0773bc61ee89c3c2791bfea1e203 3/3 column families, dataSize=161.02 KB heapSize=453.47 KB 2024-12-09T23:56:48,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=A 2024-12-09T23:56:48,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T23:56:48,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=B 2024-12-09T23:56:48,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T23:56:48,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=C 2024-12-09T23:56:48,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T23:56:48,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/7c011deb065146f0b1f95ea8abb31f92 is 50, key is test_row_0/A:col10/1733788608715/Put/seqid=0 2024-12-09T23:56:48,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:48,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788668721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:48,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:48,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:48,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788668724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:48,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:48,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:48,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788668727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:48,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:48,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:48,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788668731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:48,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:48,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:48,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788668731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:48,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:48,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741846_1022 (size=12151) 2024-12-09T23:56:48,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/7c011deb065146f0b1f95ea8abb31f92 2024-12-09T23:56:48,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/ea4b6c50d35844cd92a95822eb5e776b is 50, key is test_row_0/B:col10/1733788608715/Put/seqid=0 2024-12-09T23:56:48,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741847_1023 (size=12151) 2024-12-09T23:56:48,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/ea4b6c50d35844cd92a95822eb5e776b 2024-12-09T23:56:48,803 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-09T23:56:48,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/9df03c58c0234258a376de4fcb864992 is 50, key is test_row_0/C:col10/1733788608715/Put/seqid=0 2024-12-09T23:56:48,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741848_1024 (size=12151) 2024-12-09T23:56:48,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/9df03c58c0234258a376de4fcb864992 2024-12-09T23:56:48,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/7c011deb065146f0b1f95ea8abb31f92 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/7c011deb065146f0b1f95ea8abb31f92 2024-12-09T23:56:48,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:48,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788668843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:48,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:48,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:48,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788668844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:48,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:48,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:48,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788668844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:48,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:48,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:48,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788668844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:48,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:48,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:48,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788668853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:48,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:48,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:48,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/7c011deb065146f0b1f95ea8abb31f92, entries=150, sequenceid=164, filesize=11.9 K 2024-12-09T23:56:48,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/ea4b6c50d35844cd92a95822eb5e776b as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/ea4b6c50d35844cd92a95822eb5e776b 2024-12-09T23:56:48,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/ea4b6c50d35844cd92a95822eb5e776b, entries=150, sequenceid=164, filesize=11.9 K 2024-12-09T23:56:48,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/9df03c58c0234258a376de4fcb864992 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/9df03c58c0234258a376de4fcb864992 2024-12-09T23:56:48,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/9df03c58c0234258a376de4fcb864992, entries=150, sequenceid=164, filesize=11.9 K 2024-12-09T23:56:48,899 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~453.42 KB/464304, currentSize=26.84 KB/27480 for c1cc0773bc61ee89c3c2791bfea1e203 in 180ms, sequenceid=164, compaction requested=true 2024-12-09T23:56:48,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:56:48,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cc0773bc61ee89c3c2791bfea1e203:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:56:48,908 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T23:56:48,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:56:48,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cc0773bc61ee89c3c2791bfea1e203:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:56:48,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:56:48,909 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T23:56:48,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cc0773bc61ee89c3c2791bfea1e203:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:56:48,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:56:48,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T23:56:48,914 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:56:48,914 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48154 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T23:56:48,914 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:56:48,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-09T23:56:48,916 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): c1cc0773bc61ee89c3c2791bfea1e203/B is initiating minor compaction (all files) 2024-12-09T23:56:48,917 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cc0773bc61ee89c3c2791bfea1e203/B in TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:48,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T23:56:48,917 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/80de9e803b58411c9848a24e09f80ade, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/01b88dc5dc924475ab73054189622e83, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/6e72ff982971413bbc64347fe85f54e1, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/ea4b6c50d35844cd92a95822eb5e776b] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp, totalSize=47.0 K 2024-12-09T23:56:48,918 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:56:48,919 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 80de9e803b58411c9848a24e09f80ade, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733788606788 2024-12-09T23:56:48,920 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:56:48,920 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:56:48,920 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 01b88dc5dc924475ab73054189622e83, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733788607035 2024-12-09T23:56:48,921 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e72ff982971413bbc64347fe85f54e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1733788607414 2024-12-09T23:56:48,923 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting ea4b6c50d35844cd92a95822eb5e776b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733788608370 2024-12-09T23:56:48,929 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52834 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T23:56:48,929 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): c1cc0773bc61ee89c3c2791bfea1e203/A is initiating minor compaction (all files) 2024-12-09T23:56:48,929 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cc0773bc61ee89c3c2791bfea1e203/A in TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:48,930 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/635f28a26aea4f9bbb9cf7081136bb66, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/3b31be394fd84f488e8f169c06213427, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/e201be93a15a4e849d4f510a182bf4f4, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/7c011deb065146f0b1f95ea8abb31f92] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp, totalSize=51.6 K 2024-12-09T23:56:48,932 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 635f28a26aea4f9bbb9cf7081136bb66, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733788606788 2024-12-09T23:56:48,933 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b31be394fd84f488e8f169c06213427, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733788607035 2024-12-09T23:56:48,937 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting e201be93a15a4e849d4f510a182bf4f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1733788607414 2024-12-09T23:56:48,938 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c011deb065146f0b1f95ea8abb31f92, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733788608370 2024-12-09T23:56:48,983 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cc0773bc61ee89c3c2791bfea1e203#B#compaction#12 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:56:48,984 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/16683551c7254d00b99ad62d675b606b is 50, key is test_row_0/B:col10/1733788608715/Put/seqid=0 2024-12-09T23:56:48,985 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cc0773bc61ee89c3c2791bfea1e203#A#compaction#13 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:56:48,985 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/d8be3c94d55b4eb780ab9df802fb9118 is 50, key is test_row_0/A:col10/1733788608715/Put/seqid=0 2024-12-09T23:56:49,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741849_1025 (size=12289) 2024-12-09T23:56:49,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T23:56:49,035 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/16683551c7254d00b99ad62d675b606b as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/16683551c7254d00b99ad62d675b606b 2024-12-09T23:56:49,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741850_1026 (size=12289) 2024-12-09T23:56:49,061 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1cc0773bc61ee89c3c2791bfea1e203/B of c1cc0773bc61ee89c3c2791bfea1e203 into 16683551c7254d00b99ad62d675b606b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:56:49,061 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:56:49,068 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., storeName=c1cc0773bc61ee89c3c2791bfea1e203/B, priority=12, startTime=1733788608909; duration=0sec 2024-12-09T23:56:49,069 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:56:49,069 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cc0773bc61ee89c3c2791bfea1e203:B 2024-12-09T23:56:49,069 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T23:56:49,074 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1cc0773bc61ee89c3c2791bfea1e203 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T23:56:49,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=A 2024-12-09T23:56:49,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:56:49,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=B 2024-12-09T23:56:49,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:56:49,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=C 2024-12-09T23:56:49,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:56:49,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T23:56:49,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:49,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:56:49,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:56:49,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:49,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:49,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:49,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:49,081 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48154 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T23:56:49,082 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): c1cc0773bc61ee89c3c2791bfea1e203/C is initiating minor compaction (all files) 2024-12-09T23:56:49,082 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cc0773bc61ee89c3c2791bfea1e203/C in TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:49,082 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/7d27212aede44f18adb8034aef636699, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/34317ca0c5624706958db763e910f8a3, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/9c07739a9df14ea48f85c5034a9aac56, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/9df03c58c0234258a376de4fcb864992] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp, totalSize=47.0 K 2024-12-09T23:56:49,084 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d27212aede44f18adb8034aef636699, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733788606788 2024-12-09T23:56:49,085 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 34317ca0c5624706958db763e910f8a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733788607035 2024-12-09T23:56:49,087 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c07739a9df14ea48f85c5034a9aac56, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1733788607414 2024-12-09T23:56:49,089 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 9df03c58c0234258a376de4fcb864992, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733788608370 2024-12-09T23:56:49,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/072c8074462a4f57bb86f403a64151f9 is 50, key is test_row_0/A:col10/1733788609070/Put/seqid=0 2024-12-09T23:56:49,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733788609115, max timestamp=1733788609133Num uniques 859; 2024-12-09T23:56:49,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733788609115, max timestamp=1733788609133Num uniques 859; 2024-12-09T23:56:49,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T23:56:49,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T23:56:49,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733788609115, max timestamp=1733788609133Num uniques 859; 2024-12-09T23:56:49,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T23:56:49,168 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cc0773bc61ee89c3c2791bfea1e203#C#compaction#15 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:56:49,169 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/ae85d0fd792f46d88761cb60dedb7791 is 50, key is test_row_0/C:col10/1733788608715/Put/seqid=0 2024-12-09T23:56:49,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741851_1027 (size=14541) 2024-12-09T23:56:49,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:49,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788669168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:49,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/072c8074462a4f57bb86f403a64151f9 2024-12-09T23:56:49,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:49,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:49,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788669175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:49,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:49,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788669175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:49,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:49,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:49,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:49,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788669180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:49,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:49,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:49,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788669180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:49,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:49,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/86bb984ba4234388b1c3053deddcf95b is 50, key is test_row_0/B:col10/1733788609070/Put/seqid=0 2024-12-09T23:56:49,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741852_1028 (size=12289) 2024-12-09T23:56:49,227 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/ae85d0fd792f46d88761cb60dedb7791 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/ae85d0fd792f46d88761cb60dedb7791 2024-12-09T23:56:49,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T23:56:49,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:49,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:56:49,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:49,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:49,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:49,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:49,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T23:56:49,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741853_1029 (size=12151) 2024-12-09T23:56:49,247 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1cc0773bc61ee89c3c2791bfea1e203/C of c1cc0773bc61ee89c3c2791bfea1e203 into ae85d0fd792f46d88761cb60dedb7791(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:56:49,247 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:56:49,247 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., storeName=c1cc0773bc61ee89c3c2791bfea1e203/C, priority=12, startTime=1733788608909; duration=0sec 2024-12-09T23:56:49,247 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:56:49,247 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cc0773bc61ee89c3c2791bfea1e203:C 2024-12-09T23:56:49,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/86bb984ba4234388b1c3053deddcf95b 2024-12-09T23:56:49,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/b0d319bcd4ab4affb6abe8c048d6a5fe is 50, key is test_row_0/C:col10/1733788609070/Put/seqid=0 2024-12-09T23:56:49,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741854_1030 (size=12151) 2024-12-09T23:56:49,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:49,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788669293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:49,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:49,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:49,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788669294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:49,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:49,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:49,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788669295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:49,297 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/b0d319bcd4ab4affb6abe8c048d6a5fe 2024-12-09T23:56:49,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:49,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:49,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788669303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:49,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:49,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788669304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:49,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:49,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:49,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/072c8074462a4f57bb86f403a64151f9 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/072c8074462a4f57bb86f403a64151f9 2024-12-09T23:56:49,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/072c8074462a4f57bb86f403a64151f9, entries=200, sequenceid=175, filesize=14.2 K 2024-12-09T23:56:49,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/86bb984ba4234388b1c3053deddcf95b as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/86bb984ba4234388b1c3053deddcf95b 2024-12-09T23:56:49,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/86bb984ba4234388b1c3053deddcf95b, entries=150, sequenceid=175, filesize=11.9 K 2024-12-09T23:56:49,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/b0d319bcd4ab4affb6abe8c048d6a5fe as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/b0d319bcd4ab4affb6abe8c048d6a5fe 2024-12-09T23:56:49,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/b0d319bcd4ab4affb6abe8c048d6a5fe, entries=150, sequenceid=175, filesize=11.9 K 2024-12-09T23:56:49,359 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for c1cc0773bc61ee89c3c2791bfea1e203 in 286ms, sequenceid=175, compaction requested=false 2024-12-09T23:56:49,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:56:49,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T23:56:49,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:49,386 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing c1cc0773bc61ee89c3c2791bfea1e203 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-12-09T23:56:49,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=A 2024-12-09T23:56:49,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:56:49,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=B 2024-12-09T23:56:49,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:56:49,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=C 2024-12-09T23:56:49,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:56:49,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/4effd93271284b24a615b19edd357a87 is 50, key is test_row_0/A:col10/1733788609172/Put/seqid=0 2024-12-09T23:56:49,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741855_1031 (size=12151) 2024-12-09T23:56:49,439 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/4effd93271284b24a615b19edd357a87 2024-12-09T23:56:49,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/dbe2e15349d9454b8f6e43c3cd51cc94 is 50, key is test_row_0/B:col10/1733788609172/Put/seqid=0 2024-12-09T23:56:49,470 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/d8be3c94d55b4eb780ab9df802fb9118 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/d8be3c94d55b4eb780ab9df802fb9118 2024-12-09T23:56:49,485 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1cc0773bc61ee89c3c2791bfea1e203/A of c1cc0773bc61ee89c3c2791bfea1e203 into d8be3c94d55b4eb780ab9df802fb9118(size=12.0 K), total size for store is 26.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:56:49,485 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:56:49,486 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., storeName=c1cc0773bc61ee89c3c2791bfea1e203/A, priority=12, startTime=1733788608901; duration=0sec 2024-12-09T23:56:49,486 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:56:49,486 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cc0773bc61ee89c3c2791bfea1e203:A 2024-12-09T23:56:49,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741856_1032 (size=12151) 2024-12-09T23:56:49,495 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/dbe2e15349d9454b8f6e43c3cd51cc94 2024-12-09T23:56:49,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:56:49,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:56:49,514 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T23:56:49,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/9c88d343dd2e48289012fb6b29275f82 is 50, key is test_row_0/C:col10/1733788609172/Put/seqid=0 2024-12-09T23:56:49,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:49,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788669523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:49,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:49,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788669525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:49,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:49,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:49,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:49,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788669527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:49,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:49,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:49,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788669530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:49,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:49,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:49,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788669530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:49,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:49,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T23:56:49,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741857_1033 (size=12151) 2024-12-09T23:56:49,572 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/9c88d343dd2e48289012fb6b29275f82 2024-12-09T23:56:49,583 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-09T23:56:49,583 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-09T23:56:49,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/4effd93271284b24a615b19edd357a87 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/4effd93271284b24a615b19edd357a87 2024-12-09T23:56:49,586 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T23:56:49,586 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-09T23:56:49,586 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-09T23:56:49,586 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-09T23:56:49,588 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-09T23:56:49,588 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-09T23:56:49,597 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/4effd93271284b24a615b19edd357a87, entries=150, sequenceid=212, filesize=11.9 K 2024-12-09T23:56:49,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/dbe2e15349d9454b8f6e43c3cd51cc94 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/dbe2e15349d9454b8f6e43c3cd51cc94 2024-12-09T23:56:49,609 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/dbe2e15349d9454b8f6e43c3cd51cc94, entries=150, sequenceid=212, filesize=11.9 K 2024-12-09T23:56:49,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/9c88d343dd2e48289012fb6b29275f82 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/9c88d343dd2e48289012fb6b29275f82 2024-12-09T23:56:49,623 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/9c88d343dd2e48289012fb6b29275f82, entries=150, sequenceid=212, filesize=11.9 K 2024-12-09T23:56:49,625 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=53.67 KB/54960 for c1cc0773bc61ee89c3c2791bfea1e203 in 239ms, sequenceid=212, compaction requested=true 2024-12-09T23:56:49,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:56:49,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:49,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-09T23:56:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-09T23:56:49,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-09T23:56:49,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 708 msec 2024-12-09T23:56:49,637 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 720 msec 2024-12-09T23:56:49,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:56:49,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1cc0773bc61ee89c3c2791bfea1e203 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-09T23:56:49,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=A 2024-12-09T23:56:49,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:56:49,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=B 2024-12-09T23:56:49,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:56:49,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=C 2024-12-09T23:56:49,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:56:49,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/e4f7a6da235445639ceb3567c3b8dfb7 is 50, key is test_row_0/A:col10/1733788609645/Put/seqid=0 2024-12-09T23:56:49,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:49,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:49,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788669672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:49,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:49,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:49,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788669672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:49,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788669673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:49,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:49,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:49,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:49,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788669680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:49,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:49,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:49,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788669684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:49,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:49,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741858_1034 (size=12151) 2024-12-09T23:56:49,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/e4f7a6da235445639ceb3567c3b8dfb7 2024-12-09T23:56:49,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/b9ebb4e000bf42c196be0a5cbb407c4b is 50, key is test_row_0/B:col10/1733788609645/Put/seqid=0 2024-12-09T23:56:49,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741859_1035 (size=12151) 2024-12-09T23:56:49,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/b9ebb4e000bf42c196be0a5cbb407c4b 2024-12-09T23:56:49,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/82d9ff3835774c28b2895003afc77621 is 50, key is test_row_0/C:col10/1733788609645/Put/seqid=0 2024-12-09T23:56:49,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:49,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788669793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:49,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:49,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:49,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788669793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:49,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:49,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:49,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788669794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:49,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741860_1036 (size=12151) 2024-12-09T23:56:49,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:49,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:49,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788669794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:49,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/82d9ff3835774c28b2895003afc77621 2024-12-09T23:56:49,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:49,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:49,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788669794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:49,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:49,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:49,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/e4f7a6da235445639ceb3567c3b8dfb7 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/e4f7a6da235445639ceb3567c3b8dfb7 2024-12-09T23:56:49,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/e4f7a6da235445639ceb3567c3b8dfb7, entries=150, sequenceid=229, filesize=11.9 K 2024-12-09T23:56:49,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/b9ebb4e000bf42c196be0a5cbb407c4b as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/b9ebb4e000bf42c196be0a5cbb407c4b 2024-12-09T23:56:49,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/b9ebb4e000bf42c196be0a5cbb407c4b, entries=150, sequenceid=229, filesize=11.9 K 2024-12-09T23:56:49,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/82d9ff3835774c28b2895003afc77621 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/82d9ff3835774c28b2895003afc77621 2024-12-09T23:56:49,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/82d9ff3835774c28b2895003afc77621, entries=150, sequenceid=229, filesize=11.9 K 2024-12-09T23:56:49,859 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for c1cc0773bc61ee89c3c2791bfea1e203 in 212ms, sequenceid=229, compaction requested=true 2024-12-09T23:56:49,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:56:49,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cc0773bc61ee89c3c2791bfea1e203:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:56:49,860 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T23:56:49,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:56:49,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cc0773bc61ee89c3c2791bfea1e203:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:56:49,860 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T23:56:49,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:56:49,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cc0773bc61ee89c3c2791bfea1e203:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:56:49,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:56:49,862 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48742 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T23:56:49,862 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51132 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T23:56:49,862 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): c1cc0773bc61ee89c3c2791bfea1e203/A is initiating minor compaction (all files) 2024-12-09T23:56:49,862 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): c1cc0773bc61ee89c3c2791bfea1e203/B is initiating minor compaction (all files) 2024-12-09T23:56:49,862 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cc0773bc61ee89c3c2791bfea1e203/A in TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:49,862 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cc0773bc61ee89c3c2791bfea1e203/B in TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:49,862 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/d8be3c94d55b4eb780ab9df802fb9118, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/072c8074462a4f57bb86f403a64151f9, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/4effd93271284b24a615b19edd357a87, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/e4f7a6da235445639ceb3567c3b8dfb7] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp, totalSize=49.9 K 2024-12-09T23:56:49,862 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/16683551c7254d00b99ad62d675b606b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/86bb984ba4234388b1c3053deddcf95b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/dbe2e15349d9454b8f6e43c3cd51cc94, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/b9ebb4e000bf42c196be0a5cbb407c4b] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp, totalSize=47.6 K 2024-12-09T23:56:49,863 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 16683551c7254d00b99ad62d675b606b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733788608370 2024-12-09T23:56:49,864 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8be3c94d55b4eb780ab9df802fb9118, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733788608370 2024-12-09T23:56:49,864 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 86bb984ba4234388b1c3053deddcf95b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733788609060 2024-12-09T23:56:49,868 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 072c8074462a4f57bb86f403a64151f9, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733788608720 2024-12-09T23:56:49,868 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting dbe2e15349d9454b8f6e43c3cd51cc94, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733788609172 2024-12-09T23:56:49,868 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting b9ebb4e000bf42c196be0a5cbb407c4b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733788609645 2024-12-09T23:56:49,869 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4effd93271284b24a615b19edd357a87, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733788609172 2024-12-09T23:56:49,871 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4f7a6da235445639ceb3567c3b8dfb7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733788609645 2024-12-09T23:56:49,910 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cc0773bc61ee89c3c2791bfea1e203#A#compaction#24 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:56:49,911 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/47d5324a9de64e10a2b1ebb5da6ef060 is 50, key is test_row_0/A:col10/1733788609645/Put/seqid=0 2024-12-09T23:56:49,918 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cc0773bc61ee89c3c2791bfea1e203#B#compaction#25 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:56:49,919 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/c495ca8982e24f77a907035b3638ad36 is 50, key is test_row_0/B:col10/1733788609645/Put/seqid=0 2024-12-09T23:56:49,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741861_1037 (size=12425) 2024-12-09T23:56:49,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741862_1038 (size=12425) 2024-12-09T23:56:49,960 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/47d5324a9de64e10a2b1ebb5da6ef060 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/47d5324a9de64e10a2b1ebb5da6ef060 2024-12-09T23:56:49,968 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/c495ca8982e24f77a907035b3638ad36 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/c495ca8982e24f77a907035b3638ad36 2024-12-09T23:56:49,985 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1cc0773bc61ee89c3c2791bfea1e203/A of c1cc0773bc61ee89c3c2791bfea1e203 into 47d5324a9de64e10a2b1ebb5da6ef060(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:56:49,985 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:56:49,985 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., storeName=c1cc0773bc61ee89c3c2791bfea1e203/A, priority=12, startTime=1733788609860; duration=0sec 2024-12-09T23:56:49,985 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1cc0773bc61ee89c3c2791bfea1e203/B of c1cc0773bc61ee89c3c2791bfea1e203 into c495ca8982e24f77a907035b3638ad36(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:56:49,985 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:56:49,985 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:56:49,985 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., storeName=c1cc0773bc61ee89c3c2791bfea1e203/B, priority=12, startTime=1733788609860; duration=0sec 2024-12-09T23:56:49,985 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cc0773bc61ee89c3c2791bfea1e203:A 2024-12-09T23:56:49,985 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T23:56:49,985 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:56:49,985 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cc0773bc61ee89c3c2791bfea1e203:B 2024-12-09T23:56:49,988 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48742 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T23:56:49,989 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): c1cc0773bc61ee89c3c2791bfea1e203/C is initiating minor compaction (all files) 2024-12-09T23:56:49,989 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cc0773bc61ee89c3c2791bfea1e203/C in TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:49,989 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/ae85d0fd792f46d88761cb60dedb7791, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/b0d319bcd4ab4affb6abe8c048d6a5fe, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/9c88d343dd2e48289012fb6b29275f82, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/82d9ff3835774c28b2895003afc77621] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp, totalSize=47.6 K 2024-12-09T23:56:49,990 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae85d0fd792f46d88761cb60dedb7791, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733788608370 2024-12-09T23:56:49,991 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0d319bcd4ab4affb6abe8c048d6a5fe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733788609060 2024-12-09T23:56:49,991 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c88d343dd2e48289012fb6b29275f82, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733788609172 2024-12-09T23:56:49,992 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82d9ff3835774c28b2895003afc77621, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733788609645 2024-12-09T23:56:50,016 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1cc0773bc61ee89c3c2791bfea1e203 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-09T23:56:50,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=A 2024-12-09T23:56:50,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:56:50,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=B 2024-12-09T23:56:50,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:56:50,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=C 2024-12-09T23:56:50,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:56:50,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:56:50,023 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cc0773bc61ee89c3c2791bfea1e203#C#compaction#26 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:56:50,024 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/02106665c08846b2a70c99ec3933ddc9 is 50, key is test_row_0/C:col10/1733788609645/Put/seqid=0 2024-12-09T23:56:50,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/37285f2fc5cb4fc7a9537f1dcb4e8de2 is 50, key is test_row_0/A:col10/1733788610006/Put/seqid=0 2024-12-09T23:56:50,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:50,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788670037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:50,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788670040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:50,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:50,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:50,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788670037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:50,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:50,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:50,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788670044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:50,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:50,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:50,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788670044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:50,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T23:56:50,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:50,053 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:56:50,053 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:56:50,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-09T23:56:50,057 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:56:50,059 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:56:50,059 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:56:50,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T23:56:50,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741864_1040 (size=14541) 2024-12-09T23:56:50,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741863_1039 (size=12425) 2024-12-09T23:56:50,080 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/02106665c08846b2a70c99ec3933ddc9 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/02106665c08846b2a70c99ec3933ddc9 2024-12-09T23:56:50,093 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1cc0773bc61ee89c3c2791bfea1e203/C of c1cc0773bc61ee89c3c2791bfea1e203 into 02106665c08846b2a70c99ec3933ddc9(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:56:50,095 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:56:50,095 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., storeName=c1cc0773bc61ee89c3c2791bfea1e203/C, priority=12, startTime=1733788609860; duration=0sec 2024-12-09T23:56:50,096 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:56:50,096 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cc0773bc61ee89c3c2791bfea1e203:C 2024-12-09T23:56:50,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:50,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:50,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:50,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788670153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:50,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788670153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:50,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788670153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:50,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:50,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:50,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:50,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T23:56:50,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:50,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788670163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:50,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:50,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788670163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:50,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:50,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:50,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-09T23:56:50,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:50,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:56:50,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:50,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:50,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:50,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:50,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:50,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788670363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:50,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:50,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:50,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788670363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:50,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:50,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:50,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788670363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:50,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:50,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-09T23:56:50,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:50,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:56:50,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:50,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:50,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:50,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:50,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T23:56:50,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:50,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788670375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:50,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:50,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:50,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788670376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:50,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:50,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/37285f2fc5cb4fc7a9537f1dcb4e8de2 2024-12-09T23:56:50,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/829bf64bb7c6473c919d0ee6a724c34f is 50, key is test_row_0/B:col10/1733788610006/Put/seqid=0 2024-12-09T23:56:50,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741865_1041 (size=12151) 2024-12-09T23:56:50,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-09T23:56:50,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:50,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:56:50,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:50,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:50,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:50,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:50,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:50,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788670673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:50,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:50,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:50,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788670673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:50,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:50,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:50,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788670673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:50,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:50,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-09T23:56:50,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:50,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:56:50,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:50,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:50,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T23:56:50,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:50,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:50,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:50,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788670683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:50,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:50,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788670683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:50,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:50,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:50,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:50,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-09T23:56:50,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:50,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:56:50,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:50,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:50,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:50,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:50,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/829bf64bb7c6473c919d0ee6a724c34f 2024-12-09T23:56:50,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/1e6c5a1161d140b59d3ab1d8d115553b is 50, key is test_row_0/C:col10/1733788610006/Put/seqid=0 2024-12-09T23:56:50,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741866_1042 (size=12151) 2024-12-09T23:56:50,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/1e6c5a1161d140b59d3ab1d8d115553b 2024-12-09T23:56:50,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/37285f2fc5cb4fc7a9537f1dcb4e8de2 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/37285f2fc5cb4fc7a9537f1dcb4e8de2 2024-12-09T23:56:50,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/37285f2fc5cb4fc7a9537f1dcb4e8de2, entries=200, sequenceid=248, filesize=14.2 K 2024-12-09T23:56:50,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/829bf64bb7c6473c919d0ee6a724c34f as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/829bf64bb7c6473c919d0ee6a724c34f 2024-12-09T23:56:50,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/829bf64bb7c6473c919d0ee6a724c34f, entries=150, sequenceid=248, filesize=11.9 K 2024-12-09T23:56:50,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/1e6c5a1161d140b59d3ab1d8d115553b as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/1e6c5a1161d140b59d3ab1d8d115553b 2024-12-09T23:56:50,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-09T23:56:50,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:50,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:56:50,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:50,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:50,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:50,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:50,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/1e6c5a1161d140b59d3ab1d8d115553b, entries=150, sequenceid=248, filesize=11.9 K 2024-12-09T23:56:50,998 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c1cc0773bc61ee89c3c2791bfea1e203 in 983ms, sequenceid=248, compaction requested=false 2024-12-09T23:56:50,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:56:51,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-09T23:56:51,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:51,147 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing c1cc0773bc61ee89c3c2791bfea1e203 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-09T23:56:51,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=A 2024-12-09T23:56:51,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:56:51,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=B 2024-12-09T23:56:51,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:56:51,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=C 2024-12-09T23:56:51,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:56:51,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/b63153ee1caf46848b214f3abecdead8 is 50, key is test_row_0/A:col10/1733788610041/Put/seqid=0 2024-12-09T23:56:51,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741867_1043 (size=12251) 2024-12-09T23:56:51,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:56:51,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:56:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T23:56:51,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788671203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:51,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788671204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:51,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:51,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:51,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788671207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:51,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:51,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788671208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:51,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:51,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:51,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788671211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:51,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:51,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:51,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788671312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:51,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:51,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788671313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:51,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:51,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:51,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788671323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:51,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:51,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788671323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:51,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:51,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788671323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:51,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:51,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788671523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:51,527 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:51,527 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,527 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788671523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:51,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:51,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788671533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:51,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:51,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788671533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:51,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:51,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788671533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:51,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:51,567 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/b63153ee1caf46848b214f3abecdead8 2024-12-09T23:56:51,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/604714add4c04d6b9e9f92974a1b2f50 is 50, key is test_row_0/B:col10/1733788610041/Put/seqid=0 2024-12-09T23:56:51,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741868_1044 (size=12251) 2024-12-09T23:56:51,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:51,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:51,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788671833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:51,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788671833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:51,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:51,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:51,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:51,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788671843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:51,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:51,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788671843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:51,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:51,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788671844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:51,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:51,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:51,999 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/604714add4c04d6b9e9f92974a1b2f50 2024-12-09T23:56:52,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/2f5c5d6020824b498f4df3e2c18ac607 is 50, key is test_row_0/C:col10/1733788610041/Put/seqid=0 2024-12-09T23:56:52,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741869_1045 (size=12251) 2024-12-09T23:56:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T23:56:52,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:52,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788672343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:52,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:52,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:52,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:52,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:52,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788672343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:52,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:52,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:52,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:52,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788672353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:52,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:52,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:52,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:52,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788672353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:52,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:52,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:52,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:52,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788672354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:52,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:52,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:52,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:52,430 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/2f5c5d6020824b498f4df3e2c18ac607 2024-12-09T23:56:52,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/b63153ee1caf46848b214f3abecdead8 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/b63153ee1caf46848b214f3abecdead8 2024-12-09T23:56:52,448 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/b63153ee1caf46848b214f3abecdead8, entries=150, sequenceid=264, filesize=12.0 K 2024-12-09T23:56:52,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/604714add4c04d6b9e9f92974a1b2f50 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/604714add4c04d6b9e9f92974a1b2f50 2024-12-09T23:56:52,463 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/604714add4c04d6b9e9f92974a1b2f50, entries=150, sequenceid=264, filesize=12.0 K 2024-12-09T23:56:52,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/2f5c5d6020824b498f4df3e2c18ac607 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/2f5c5d6020824b498f4df3e2c18ac607 2024-12-09T23:56:52,475 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/2f5c5d6020824b498f4df3e2c18ac607, entries=150, sequenceid=264, filesize=12.0 K 2024-12-09T23:56:52,481 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c1cc0773bc61ee89c3c2791bfea1e203 in 1333ms, sequenceid=264, compaction requested=true 2024-12-09T23:56:52,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:56:52,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:52,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-09T23:56:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-09T23:56:52,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-09T23:56:52,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4240 sec 2024-12-09T23:56:52,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.4350 sec 2024-12-09T23:56:53,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:56:53,356 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1cc0773bc61ee89c3c2791bfea1e203 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-09T23:56:53,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=A 2024-12-09T23:56:53,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:56:53,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=B 2024-12-09T23:56:53,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:56:53,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=C 2024-12-09T23:56:53,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:56:53,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/e0b1e0d2822e4b79a253df2598869092 is 50, key is test_row_0/A:col10/1733788611198/Put/seqid=0 2024-12-09T23:56:53,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741870_1046 (size=17181) 2024-12-09T23:56:53,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/e0b1e0d2822e4b79a253df2598869092 2024-12-09T23:56:53,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/4bf9353b8da24d87a28682059c103f6d is 50, key is test_row_0/B:col10/1733788611198/Put/seqid=0 2024-12-09T23:56:53,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:53,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:53,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788673380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:53,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788673380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:53,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:53,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:53,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:53,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788673382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:53,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:53,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:53,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:53,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788673383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:53,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:53,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:53,385 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:53,385 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:53,385 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:53,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:53,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788673384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:53,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:53,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:53,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:53,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:53,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:53,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:53,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741871_1047 (size=12301) 2024-12-09T23:56:53,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:53,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788673492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:53,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:53,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:53,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:53,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:53,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788673493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:53,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:53,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788673493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:53,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:53,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:53,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:53,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:53,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788673493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:53,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:53,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:53,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:53,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:53,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:53,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:53,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:53,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788673493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:53,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:53,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:53,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:53,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:53,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788673703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:53,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:53,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:53,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:53,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:53,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788673703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:53,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:53,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:53,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:53,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:53,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788673703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:53,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:53,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:53,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:53,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:53,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:53,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788673703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:53,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788673704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:53,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:53,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:53,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:53,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:53,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:53,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:53,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/4bf9353b8da24d87a28682059c103f6d 2024-12-09T23:56:53,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/37b87d3be795411f9ca2b277f5c526c7 is 50, key is test_row_0/C:col10/1733788611198/Put/seqid=0 2024-12-09T23:56:53,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741872_1048 (size=12301) 2024-12-09T23:56:54,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:54,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788674012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:54,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:54,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:54,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788674012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:54,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:54,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788674012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:54,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:54,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:54,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:54,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:54,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788674013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:54,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788674013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:54,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:54,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:54,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/37b87d3be795411f9ca2b277f5c526c7 2024-12-09T23:56:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T23:56:54,223 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:56:54,223 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:56:54,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-09T23:56:54,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T23:56:54,258 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:56:54,259 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:56:54,260 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:56:54,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/e0b1e0d2822e4b79a253df2598869092 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/e0b1e0d2822e4b79a253df2598869092 2024-12-09T23:56:54,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/e0b1e0d2822e4b79a253df2598869092, entries=250, sequenceid=283, filesize=16.8 K 2024-12-09T23:56:54,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/4bf9353b8da24d87a28682059c103f6d as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/4bf9353b8da24d87a28682059c103f6d 2024-12-09T23:56:54,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/4bf9353b8da24d87a28682059c103f6d, entries=150, sequenceid=283, filesize=12.0 K 2024-12-09T23:56:54,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/37b87d3be795411f9ca2b277f5c526c7 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/37b87d3be795411f9ca2b277f5c526c7 2024-12-09T23:56:54,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/37b87d3be795411f9ca2b277f5c526c7, entries=150, sequenceid=283, filesize=12.0 K 2024-12-09T23:56:54,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c1cc0773bc61ee89c3c2791bfea1e203 in 931ms, sequenceid=283, compaction requested=true 2024-12-09T23:56:54,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:56:54,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cc0773bc61ee89c3c2791bfea1e203:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:56:54,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:56:54,288 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T23:56:54,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cc0773bc61ee89c3c2791bfea1e203:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:56:54,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:56:54,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cc0773bc61ee89c3c2791bfea1e203:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:56:54,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T23:56:54,288 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T23:56:54,290 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56398 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T23:56:54,290 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): c1cc0773bc61ee89c3c2791bfea1e203/A is initiating minor compaction (all files) 2024-12-09T23:56:54,290 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cc0773bc61ee89c3c2791bfea1e203/A in TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:54,290 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49128 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T23:56:54,290 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): c1cc0773bc61ee89c3c2791bfea1e203/B is initiating minor compaction (all files) 2024-12-09T23:56:54,291 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cc0773bc61ee89c3c2791bfea1e203/B in TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:54,291 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/c495ca8982e24f77a907035b3638ad36, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/829bf64bb7c6473c919d0ee6a724c34f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/604714add4c04d6b9e9f92974a1b2f50, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/4bf9353b8da24d87a28682059c103f6d] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp, totalSize=48.0 K 2024-12-09T23:56:54,291 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/47d5324a9de64e10a2b1ebb5da6ef060, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/37285f2fc5cb4fc7a9537f1dcb4e8de2, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/b63153ee1caf46848b214f3abecdead8, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/e0b1e0d2822e4b79a253df2598869092] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp, totalSize=55.1 K 2024-12-09T23:56:54,292 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting c495ca8982e24f77a907035b3638ad36, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733788609645 2024-12-09T23:56:54,292 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47d5324a9de64e10a2b1ebb5da6ef060, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733788609645 2024-12-09T23:56:54,293 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 829bf64bb7c6473c919d0ee6a724c34f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733788609655 2024-12-09T23:56:54,293 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37285f2fc5cb4fc7a9537f1dcb4e8de2, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733788609655 2024-12-09T23:56:54,293 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 604714add4c04d6b9e9f92974a1b2f50, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733788610026 2024-12-09T23:56:54,294 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting b63153ee1caf46848b214f3abecdead8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733788610026 2024-12-09T23:56:54,294 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bf9353b8da24d87a28682059c103f6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733788611198 2024-12-09T23:56:54,294 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0b1e0d2822e4b79a253df2598869092, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733788611198 2024-12-09T23:56:54,308 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cc0773bc61ee89c3c2791bfea1e203#A#compaction#36 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:56:54,309 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cc0773bc61ee89c3c2791bfea1e203#B#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:56:54,309 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/5a25a8684b2947d281267fe6033be094 is 50, key is test_row_0/A:col10/1733788611198/Put/seqid=0 2024-12-09T23:56:54,309 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/8cdb31daff61487c8f83dd093c649cdd is 50, key is test_row_0/B:col10/1733788611198/Put/seqid=0 2024-12-09T23:56:54,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741874_1050 (size=12711) 2024-12-09T23:56:54,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741873_1049 (size=12711) 2024-12-09T23:56:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T23:56:54,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-09T23:56:54,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:54,413 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing c1cc0773bc61ee89c3c2791bfea1e203 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T23:56:54,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=A 2024-12-09T23:56:54,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:56:54,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=B 2024-12-09T23:56:54,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:56:54,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=C 2024-12-09T23:56:54,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:56:54,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/863c4f5fd3ea4dcfa87cfeec2f36c032 is 50, key is test_row_0/A:col10/1733788613378/Put/seqid=0 2024-12-09T23:56:54,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741875_1051 (size=12301) 2024-12-09T23:56:54,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:56:54,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:56:54,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:54,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:54,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788674542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:54,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788674542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:54,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:54,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:54,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:54,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788674543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:54,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:54,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:54,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788674547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:54,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:54,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:54,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788674547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:54,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:54,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T23:56:54,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788674653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:54,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788674653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:54,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:54,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:54,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788674653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:54,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:54,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:54,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788674663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:54,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:54,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:54,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788674663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:54,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:54,730 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/5a25a8684b2947d281267fe6033be094 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/5a25a8684b2947d281267fe6033be094 2024-12-09T23:56:54,732 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/8cdb31daff61487c8f83dd093c649cdd as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/8cdb31daff61487c8f83dd093c649cdd 2024-12-09T23:56:54,741 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1cc0773bc61ee89c3c2791bfea1e203/B of c1cc0773bc61ee89c3c2791bfea1e203 into 8cdb31daff61487c8f83dd093c649cdd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:56:54,741 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:56:54,741 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., storeName=c1cc0773bc61ee89c3c2791bfea1e203/B, priority=12, startTime=1733788614288; duration=0sec 2024-12-09T23:56:54,741 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:56:54,741 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cc0773bc61ee89c3c2791bfea1e203:B 2024-12-09T23:56:54,741 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T23:56:54,744 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49128 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T23:56:54,744 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): c1cc0773bc61ee89c3c2791bfea1e203/C is initiating minor compaction (all files) 2024-12-09T23:56:54,744 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cc0773bc61ee89c3c2791bfea1e203/C in TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:54,745 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/02106665c08846b2a70c99ec3933ddc9, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/1e6c5a1161d140b59d3ab1d8d115553b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/2f5c5d6020824b498f4df3e2c18ac607, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/37b87d3be795411f9ca2b277f5c526c7] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp, totalSize=48.0 K 2024-12-09T23:56:54,745 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 02106665c08846b2a70c99ec3933ddc9, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733788609645 2024-12-09T23:56:54,745 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1cc0773bc61ee89c3c2791bfea1e203/A of c1cc0773bc61ee89c3c2791bfea1e203 into 5a25a8684b2947d281267fe6033be094(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:56:54,745 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:56:54,745 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., storeName=c1cc0773bc61ee89c3c2791bfea1e203/A, priority=12, startTime=1733788614287; duration=0sec 2024-12-09T23:56:54,746 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:56:54,746 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cc0773bc61ee89c3c2791bfea1e203:A 2024-12-09T23:56:54,746 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e6c5a1161d140b59d3ab1d8d115553b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733788609655 2024-12-09T23:56:54,746 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f5c5d6020824b498f4df3e2c18ac607, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733788610026 2024-12-09T23:56:54,746 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 37b87d3be795411f9ca2b277f5c526c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733788611198 2024-12-09T23:56:54,759 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cc0773bc61ee89c3c2791bfea1e203#C#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:56:54,760 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/6765a71eb8794558b2a177f60c1e88bf is 50, key is test_row_0/C:col10/1733788611198/Put/seqid=0 2024-12-09T23:56:54,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741876_1052 (size=12711) 2024-12-09T23:56:54,774 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/6765a71eb8794558b2a177f60c1e88bf as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/6765a71eb8794558b2a177f60c1e88bf 2024-12-09T23:56:54,781 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1cc0773bc61ee89c3c2791bfea1e203/C of c1cc0773bc61ee89c3c2791bfea1e203 into 6765a71eb8794558b2a177f60c1e88bf(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:56:54,781 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:56:54,781 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., storeName=c1cc0773bc61ee89c3c2791bfea1e203/C, priority=12, startTime=1733788614288; duration=0sec 2024-12-09T23:56:54,782 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:56:54,782 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cc0773bc61ee89c3c2791bfea1e203:C 2024-12-09T23:56:54,831 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/863c4f5fd3ea4dcfa87cfeec2f36c032 2024-12-09T23:56:54,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/b77050f8cd414467a530b6609055600c is 50, key is test_row_0/B:col10/1733788613378/Put/seqid=0 2024-12-09T23:56:54,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741877_1053 (size=12301) 2024-12-09T23:56:54,858 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/b77050f8cd414467a530b6609055600c 2024-12-09T23:56:54,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788674863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:54,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:54,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788674865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:54,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:54,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788674865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:54,867 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,868 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,868 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:54,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/422ff75ace6f4858bd7ccd1f135d4405 is 50, key is test_row_0/C:col10/1733788613378/Put/seqid=0 2024-12-09T23:56:54,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788674873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:54,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:54,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788674875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:54,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:54,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T23:56:54,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741878_1054 (size=12301) 2024-12-09T23:56:54,892 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/422ff75ace6f4858bd7ccd1f135d4405 2024-12-09T23:56:54,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/863c4f5fd3ea4dcfa87cfeec2f36c032 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/863c4f5fd3ea4dcfa87cfeec2f36c032 2024-12-09T23:56:54,914 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/863c4f5fd3ea4dcfa87cfeec2f36c032, entries=150, sequenceid=296, filesize=12.0 K 2024-12-09T23:56:54,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/b77050f8cd414467a530b6609055600c as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/b77050f8cd414467a530b6609055600c 2024-12-09T23:56:54,923 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/b77050f8cd414467a530b6609055600c, entries=150, sequenceid=296, filesize=12.0 K 2024-12-09T23:56:54,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/422ff75ace6f4858bd7ccd1f135d4405 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/422ff75ace6f4858bd7ccd1f135d4405 2024-12-09T23:56:54,933 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/422ff75ace6f4858bd7ccd1f135d4405, entries=150, sequenceid=296, filesize=12.0 K 2024-12-09T23:56:54,935 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for c1cc0773bc61ee89c3c2791bfea1e203 in 522ms, sequenceid=296, compaction requested=false 2024-12-09T23:56:54,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:56:54,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:54,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-09T23:56:54,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-09T23:56:54,942 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-09T23:56:54,942 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 677 msec 2024-12-09T23:56:54,946 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 720 msec 2024-12-09T23:56:55,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:56:55,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1cc0773bc61ee89c3c2791bfea1e203 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-09T23:56:55,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=A 2024-12-09T23:56:55,176 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733788614533, max timestamp=1733788615174Num uniques 859; 2024-12-09T23:56:55,176 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T23:56:55,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:56:55,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=B 2024-12-09T23:56:55,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:56:55,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=C 2024-12-09T23:56:55,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:56:55,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/3fb385eddd164eec9af01ec68c430fa4 is 50, key is test_row_0/A:col10/1733788614540/Put/seqid=0 2024-12-09T23:56:55,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:55,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788675192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:55,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:55,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:55,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788675193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:55,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:55,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788675193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:55,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:55,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:55,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788675194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:55,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:55,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:55,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788675196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:55,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:55,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741879_1055 (size=12301) 2024-12-09T23:56:55,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:55,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:55,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:55,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788675302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:55,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788675303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:55,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:55,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:55,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788675303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:55,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:55,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:55,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788675303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:55,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:55,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:55,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:55,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788675313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:55,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:55,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T23:56:55,394 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:56:55,394 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:56:55,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-09T23:56:55,396 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:56:55,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T23:56:55,397 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:56:55,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:56:55,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T23:56:55,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:55,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788675512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:55,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:55,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:55,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788675512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:55,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:55,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:55,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788675513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:55,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:55,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:55,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788675523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:55,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:55,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788675523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:55,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:55,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:55,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-09T23:56:55,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:55,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:56:55,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:55,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:55,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:55,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/3fb385eddd164eec9af01ec68c430fa4 2024-12-09T23:56:55,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/3dcdea4cf6ba4696978a837e15150266 is 50, key is test_row_0/B:col10/1733788614540/Put/seqid=0 2024-12-09T23:56:55,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741880_1056 (size=12301) 2024-12-09T23:56:55,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-09T23:56:55,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:55,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:56:55,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:55,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:55,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:55,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T23:56:55,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:55,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788675823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788675823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:55,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:55,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788675824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:55,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:55,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:55,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788675833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:55,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:55,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:55,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788675833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:55,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:55,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:55,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-09T23:56:55,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:55,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:56:55,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:55,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:55,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:55,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:56,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-09T23:56:56,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:56,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:56:56,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:56,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:56,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:56,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:56,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/3dcdea4cf6ba4696978a837e15150266 2024-12-09T23:56:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T23:56:56,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/00743dc7048145f886e636466cde182f is 50, key is test_row_0/C:col10/1733788614540/Put/seqid=0 2024-12-09T23:56:56,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741881_1057 (size=12301) 2024-12-09T23:56:56,073 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/00743dc7048145f886e636466cde182f 2024-12-09T23:56:56,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/3fb385eddd164eec9af01ec68c430fa4 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/3fb385eddd164eec9af01ec68c430fa4 2024-12-09T23:56:56,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/3fb385eddd164eec9af01ec68c430fa4, entries=150, sequenceid=320, filesize=12.0 K 2024-12-09T23:56:56,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/3dcdea4cf6ba4696978a837e15150266 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/3dcdea4cf6ba4696978a837e15150266 2024-12-09T23:56:56,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/3dcdea4cf6ba4696978a837e15150266, entries=150, sequenceid=320, filesize=12.0 K 2024-12-09T23:56:56,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/00743dc7048145f886e636466cde182f as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/00743dc7048145f886e636466cde182f 2024-12-09T23:56:56,119 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/00743dc7048145f886e636466cde182f, entries=150, sequenceid=320, filesize=12.0 K 2024-12-09T23:56:56,120 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~89.45 KB/91600, heapSize ~262.06 KB/268352, currentSize=87.22 KB/89310 for c1cc0773bc61ee89c3c2791bfea1e203 in 945ms, sequenceid=320, compaction requested=true 2024-12-09T23:56:56,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:56:56,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cc0773bc61ee89c3c2791bfea1e203:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:56:56,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:56:56,120 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:56:56,120 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:56:56,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cc0773bc61ee89c3c2791bfea1e203:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:56:56,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:56:56,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cc0773bc61ee89c3c2791bfea1e203:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:56:56,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:56:56,122 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:56:56,122 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:56:56,123 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): c1cc0773bc61ee89c3c2791bfea1e203/B is initiating minor compaction (all files) 2024-12-09T23:56:56,123 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): c1cc0773bc61ee89c3c2791bfea1e203/A is initiating minor compaction (all files) 2024-12-09T23:56:56,123 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cc0773bc61ee89c3c2791bfea1e203/A in TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:56,123 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cc0773bc61ee89c3c2791bfea1e203/B in TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:56,123 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/8cdb31daff61487c8f83dd093c649cdd, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/b77050f8cd414467a530b6609055600c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/3dcdea4cf6ba4696978a837e15150266] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp, totalSize=36.4 K 2024-12-09T23:56:56,123 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/5a25a8684b2947d281267fe6033be094, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/863c4f5fd3ea4dcfa87cfeec2f36c032, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/3fb385eddd164eec9af01ec68c430fa4] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp, totalSize=36.4 K 2024-12-09T23:56:56,124 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a25a8684b2947d281267fe6033be094, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733788611198 2024-12-09T23:56:56,124 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cdb31daff61487c8f83dd093c649cdd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733788611198 2024-12-09T23:56:56,124 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 863c4f5fd3ea4dcfa87cfeec2f36c032, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733788613375 2024-12-09T23:56:56,125 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting b77050f8cd414467a530b6609055600c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733788613375 2024-12-09T23:56:56,125 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fb385eddd164eec9af01ec68c430fa4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733788614533 2024-12-09T23:56:56,126 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 3dcdea4cf6ba4696978a837e15150266, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733788614533 2024-12-09T23:56:56,137 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cc0773bc61ee89c3c2791bfea1e203#A#compaction#45 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:56:56,137 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/6b1fcd8d61494a129911ee981010afd7 is 50, key is test_row_0/A:col10/1733788614540/Put/seqid=0 2024-12-09T23:56:56,145 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cc0773bc61ee89c3c2791bfea1e203#B#compaction#46 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:56:56,146 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/c5278d28aed74da7a8cbca3a647f06c1 is 50, key is test_row_0/B:col10/1733788614540/Put/seqid=0 2024-12-09T23:56:56,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741882_1058 (size=12813) 2024-12-09T23:56:56,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741883_1059 (size=12813) 2024-12-09T23:56:56,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-09T23:56:56,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:56,168 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing c1cc0773bc61ee89c3c2791bfea1e203 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-09T23:56:56,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=A 2024-12-09T23:56:56,169 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/6b1fcd8d61494a129911ee981010afd7 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/6b1fcd8d61494a129911ee981010afd7 2024-12-09T23:56:56,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:56:56,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=B 2024-12-09T23:56:56,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:56:56,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=C 2024-12-09T23:56:56,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:56:56,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/c97d3c65c60e49069215a23776b2ab84 is 50, key is test_row_0/A:col10/1733788615194/Put/seqid=0 2024-12-09T23:56:56,180 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1cc0773bc61ee89c3c2791bfea1e203/A of c1cc0773bc61ee89c3c2791bfea1e203 into 6b1fcd8d61494a129911ee981010afd7(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:56:56,180 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:56:56,180 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., storeName=c1cc0773bc61ee89c3c2791bfea1e203/A, priority=13, startTime=1733788616120; duration=0sec 2024-12-09T23:56:56,180 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:56:56,180 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cc0773bc61ee89c3c2791bfea1e203:A 2024-12-09T23:56:56,180 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:56:56,183 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:56:56,184 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): c1cc0773bc61ee89c3c2791bfea1e203/C is initiating minor compaction (all files) 2024-12-09T23:56:56,184 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cc0773bc61ee89c3c2791bfea1e203/C in TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:56,184 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/6765a71eb8794558b2a177f60c1e88bf, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/422ff75ace6f4858bd7ccd1f135d4405, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/00743dc7048145f886e636466cde182f] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp, totalSize=36.4 K 2024-12-09T23:56:56,185 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6765a71eb8794558b2a177f60c1e88bf, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733788611198 2024-12-09T23:56:56,185 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 422ff75ace6f4858bd7ccd1f135d4405, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733788613375 2024-12-09T23:56:56,185 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00743dc7048145f886e636466cde182f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733788614533 2024-12-09T23:56:56,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741884_1060 (size=12301) 2024-12-09T23:56:56,195 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cc0773bc61ee89c3c2791bfea1e203#C#compaction#48 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:56:56,196 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/0cdb3add422045c8bd9b8197450f2ed8 is 50, key is test_row_0/C:col10/1733788614540/Put/seqid=0 2024-12-09T23:56:56,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741885_1061 (size=12813) 2024-12-09T23:56:56,209 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/0cdb3add422045c8bd9b8197450f2ed8 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/0cdb3add422045c8bd9b8197450f2ed8 2024-12-09T23:56:56,218 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1cc0773bc61ee89c3c2791bfea1e203/C of c1cc0773bc61ee89c3c2791bfea1e203 into 0cdb3add422045c8bd9b8197450f2ed8(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:56:56,218 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:56:56,218 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., storeName=c1cc0773bc61ee89c3c2791bfea1e203/C, priority=13, startTime=1733788616122; duration=0sec 2024-12-09T23:56:56,218 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:56:56,218 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cc0773bc61ee89c3c2791bfea1e203:C 2024-12-09T23:56:56,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:56:56,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:56:56,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:56,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788676351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:56,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:56,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788676353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:56,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:56,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:56,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788676354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:56,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:56,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:56,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788676356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:56,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:56,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:56,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788676356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:56,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:56,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:56,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788676463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:56,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:56,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:56,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788676463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:56,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:56,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:56,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788676463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:56,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:56,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:56,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:56,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788676473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:56,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788676473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:56,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:56,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:56,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T23:56:56,573 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/c5278d28aed74da7a8cbca3a647f06c1 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/c5278d28aed74da7a8cbca3a647f06c1 2024-12-09T23:56:56,580 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1cc0773bc61ee89c3c2791bfea1e203/B of c1cc0773bc61ee89c3c2791bfea1e203 into c5278d28aed74da7a8cbca3a647f06c1(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:56:56,580 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:56:56,580 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., storeName=c1cc0773bc61ee89c3c2791bfea1e203/B, priority=13, startTime=1733788616120; duration=0sec 2024-12-09T23:56:56,580 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:56:56,581 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cc0773bc61ee89c3c2791bfea1e203:B 2024-12-09T23:56:56,590 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/c97d3c65c60e49069215a23776b2ab84 2024-12-09T23:56:56,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/813de5dff76243bda587f1507094e889 is 50, key is test_row_0/B:col10/1733788615194/Put/seqid=0 2024-12-09T23:56:56,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741886_1062 (size=12301) 2024-12-09T23:56:56,611 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/813de5dff76243bda587f1507094e889 2024-12-09T23:56:56,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/65badc2a4c4441bab56ce32b344a214b is 50, key is test_row_0/C:col10/1733788615194/Put/seqid=0 2024-12-09T23:56:56,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741887_1063 (size=12301) 2024-12-09T23:56:56,635 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/65badc2a4c4441bab56ce32b344a214b 2024-12-09T23:56:56,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/c97d3c65c60e49069215a23776b2ab84 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/c97d3c65c60e49069215a23776b2ab84 2024-12-09T23:56:56,649 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/c97d3c65c60e49069215a23776b2ab84, entries=150, sequenceid=336, filesize=12.0 K 2024-12-09T23:56:56,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/813de5dff76243bda587f1507094e889 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/813de5dff76243bda587f1507094e889 2024-12-09T23:56:56,657 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/813de5dff76243bda587f1507094e889, entries=150, sequenceid=336, filesize=12.0 K 2024-12-09T23:56:56,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/65badc2a4c4441bab56ce32b344a214b as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/65badc2a4c4441bab56ce32b344a214b 2024-12-09T23:56:56,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788676673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:56,674 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/65badc2a4c4441bab56ce32b344a214b, entries=150, sequenceid=336, filesize=12.0 K 2024-12-09T23:56:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:56,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:56,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788676673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:56,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:56,676 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for c1cc0773bc61ee89c3c2791bfea1e203 in 508ms, sequenceid=336, compaction requested=false 2024-12-09T23:56:56,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:56:56,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:56,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-09T23:56:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:56:56,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1cc0773bc61ee89c3c2791bfea1e203 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-09T23:56:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-09T23:56:56,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=A 2024-12-09T23:56:56,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:56:56,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=B 2024-12-09T23:56:56,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:56:56,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=C 2024-12-09T23:56:56,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:56:56,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/62b20da770ee4afeaf7d23238e9657f1 is 50, key is test_row_0/A:col10/1733788616353/Put/seqid=0 2024-12-09T23:56:56,687 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-09T23:56:56,687 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2810 sec 2024-12-09T23:56:56,690 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.2940 sec 2024-12-09T23:56:56,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741888_1064 (size=14741) 2024-12-09T23:56:56,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:56,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788676701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:56,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:56,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788676706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:56,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:56,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788676707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:56,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:56,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:56,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788676813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:56,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:56,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:56,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788676823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:56,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:56,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:56,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788676823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:56,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:56,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:56,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788676983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:56,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:56,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:56,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788676983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:56,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:56,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:57,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:57,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788677022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:57,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:57,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:57,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:57,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:57,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788677033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:57,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:57,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:57,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:57,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788677033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:57,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:57,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:57,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:57,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/62b20da770ee4afeaf7d23238e9657f1 2024-12-09T23:56:57,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/eaa13bb5911a4c3a8a658fa6adffae32 is 50, key is test_row_0/B:col10/1733788616353/Put/seqid=0 2024-12-09T23:56:57,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741889_1065 (size=12301) 2024-12-09T23:56:57,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/eaa13bb5911a4c3a8a658fa6adffae32 2024-12-09T23:56:57,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/f8f11dbb1011486aa0c068cf2ae6399b is 50, key is test_row_0/C:col10/1733788616353/Put/seqid=0 2024-12-09T23:56:57,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741890_1066 (size=12301) 2024-12-09T23:56:57,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788677333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:57,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:57,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:57,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:57,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:57,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788677343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788677342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:57,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:57,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:57,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788677493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:57,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788677493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:57,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:57,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:57,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:57,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:57,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:57,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:57,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/f8f11dbb1011486aa0c068cf2ae6399b 2024-12-09T23:56:57,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T23:56:57,543 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:56:57,544 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:56:57,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-09T23:56:57,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-09T23:56:57,546 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:56:57,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/62b20da770ee4afeaf7d23238e9657f1 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/62b20da770ee4afeaf7d23238e9657f1 2024-12-09T23:56:57,547 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:56:57,547 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:56:57,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/62b20da770ee4afeaf7d23238e9657f1, entries=200, sequenceid=356, filesize=14.4 K 2024-12-09T23:56:57,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/eaa13bb5911a4c3a8a658fa6adffae32 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/eaa13bb5911a4c3a8a658fa6adffae32 2024-12-09T23:56:57,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/eaa13bb5911a4c3a8a658fa6adffae32, entries=150, sequenceid=356, filesize=12.0 K 2024-12-09T23:56:57,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/f8f11dbb1011486aa0c068cf2ae6399b as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/f8f11dbb1011486aa0c068cf2ae6399b 2024-12-09T23:56:57,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/f8f11dbb1011486aa0c068cf2ae6399b, entries=150, sequenceid=356, filesize=12.0 K 2024-12-09T23:56:57,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c1cc0773bc61ee89c3c2791bfea1e203 in 892ms, sequenceid=356, compaction requested=true 2024-12-09T23:56:57,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:56:57,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cc0773bc61ee89c3c2791bfea1e203:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:56:57,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:56:57,570 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:56:57,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cc0773bc61ee89c3c2791bfea1e203:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:56:57,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:56:57,570 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:56:57,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cc0773bc61ee89c3c2791bfea1e203:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:56:57,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:56:57,571 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:56:57,572 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): c1cc0773bc61ee89c3c2791bfea1e203/A is initiating minor compaction (all files) 2024-12-09T23:56:57,572 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cc0773bc61ee89c3c2791bfea1e203/A in TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:57,572 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/6b1fcd8d61494a129911ee981010afd7, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/c97d3c65c60e49069215a23776b2ab84, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/62b20da770ee4afeaf7d23238e9657f1] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp, totalSize=38.9 K 2024-12-09T23:56:57,572 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:56:57,572 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): c1cc0773bc61ee89c3c2791bfea1e203/B is initiating minor compaction (all files) 2024-12-09T23:56:57,572 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cc0773bc61ee89c3c2791bfea1e203/B in TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:57,572 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b1fcd8d61494a129911ee981010afd7, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733788614533 2024-12-09T23:56:57,572 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/c5278d28aed74da7a8cbca3a647f06c1, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/813de5dff76243bda587f1507094e889, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/eaa13bb5911a4c3a8a658fa6adffae32] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp, totalSize=36.5 K 2024-12-09T23:56:57,573 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting c5278d28aed74da7a8cbca3a647f06c1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733788614533 2024-12-09T23:56:57,573 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting c97d3c65c60e49069215a23776b2ab84, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1733788615189 2024-12-09T23:56:57,573 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 813de5dff76243bda587f1507094e889, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1733788615189 2024-12-09T23:56:57,573 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62b20da770ee4afeaf7d23238e9657f1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1733788616349 2024-12-09T23:56:57,574 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting eaa13bb5911a4c3a8a658fa6adffae32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1733788616353 2024-12-09T23:56:57,592 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cc0773bc61ee89c3c2791bfea1e203#B#compaction#54 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:56:57,592 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/3ebd779e73e5460d8c4c588d44c5a746 is 50, key is test_row_0/B:col10/1733788616353/Put/seqid=0 2024-12-09T23:56:57,592 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cc0773bc61ee89c3c2791bfea1e203#A#compaction#55 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:56:57,593 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/4d2af723ad5d4c348340f68f6baac4d2 is 50, key is test_row_0/A:col10/1733788616353/Put/seqid=0 2024-12-09T23:56:57,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741891_1067 (size=12915) 2024-12-09T23:56:57,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741892_1068 (size=12915) 2024-12-09T23:56:57,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-09T23:56:57,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-09T23:56:57,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:57,700 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing c1cc0773bc61ee89c3c2791bfea1e203 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-09T23:56:57,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=A 2024-12-09T23:56:57,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:56:57,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=B 2024-12-09T23:56:57,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:56:57,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=C 2024-12-09T23:56:57,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:56:57,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/2199f0d158e34811b1aa396038ad5642 is 50, key is test_row_0/A:col10/1733788616704/Put/seqid=0 2024-12-09T23:56:57,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741893_1069 (size=12301) 2024-12-09T23:56:57,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:56:57,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:56:57,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-09T23:56:57,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:57,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788677880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:57,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:57,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788677880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:57,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:57,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:57,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:57,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:57,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788677881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:57,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:57,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:57,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:57,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:57,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:57,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:58,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:58,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788678004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:58,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:58,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:58,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:58,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:58,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788678005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:58,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:58,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:58,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:58,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:58,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788678005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:58,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:58,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:58,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:58,013 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/3ebd779e73e5460d8c4c588d44c5a746 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/3ebd779e73e5460d8c4c588d44c5a746 2024-12-09T23:56:58,015 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/4d2af723ad5d4c348340f68f6baac4d2 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/4d2af723ad5d4c348340f68f6baac4d2 2024-12-09T23:56:58,023 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1cc0773bc61ee89c3c2791bfea1e203/B of c1cc0773bc61ee89c3c2791bfea1e203 into 3ebd779e73e5460d8c4c588d44c5a746(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:56:58,023 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1cc0773bc61ee89c3c2791bfea1e203/A of c1cc0773bc61ee89c3c2791bfea1e203 into 4d2af723ad5d4c348340f68f6baac4d2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:56:58,023 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:56:58,023 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:56:58,023 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., storeName=c1cc0773bc61ee89c3c2791bfea1e203/B, priority=13, startTime=1733788617570; duration=0sec 2024-12-09T23:56:58,023 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., storeName=c1cc0773bc61ee89c3c2791bfea1e203/A, priority=13, startTime=1733788617570; duration=0sec 2024-12-09T23:56:58,023 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:56:58,023 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cc0773bc61ee89c3c2791bfea1e203:A 2024-12-09T23:56:58,023 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:56:58,023 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cc0773bc61ee89c3c2791bfea1e203:B 2024-12-09T23:56:58,023 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:56:58,025 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:56:58,025 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): c1cc0773bc61ee89c3c2791bfea1e203/C is initiating minor compaction (all files) 2024-12-09T23:56:58,025 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cc0773bc61ee89c3c2791bfea1e203/C in TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:58,025 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/0cdb3add422045c8bd9b8197450f2ed8, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/65badc2a4c4441bab56ce32b344a214b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/f8f11dbb1011486aa0c068cf2ae6399b] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp, totalSize=36.5 K 2024-12-09T23:56:58,026 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cdb3add422045c8bd9b8197450f2ed8, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733788614533 2024-12-09T23:56:58,026 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65badc2a4c4441bab56ce32b344a214b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1733788615189 2024-12-09T23:56:58,026 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8f11dbb1011486aa0c068cf2ae6399b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1733788616353 2024-12-09T23:56:58,035 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cc0773bc61ee89c3c2791bfea1e203#C#compaction#57 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:56:58,036 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/150e3281f14a481fb8885e7224e2bd4f is 50, key is test_row_0/C:col10/1733788616353/Put/seqid=0 2024-12-09T23:56:58,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741894_1070 (size=12915) 2024-12-09T23:56:58,052 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/150e3281f14a481fb8885e7224e2bd4f as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/150e3281f14a481fb8885e7224e2bd4f 2024-12-09T23:56:58,059 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1cc0773bc61ee89c3c2791bfea1e203/C of c1cc0773bc61ee89c3c2791bfea1e203 into 150e3281f14a481fb8885e7224e2bd4f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:56:58,059 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:56:58,060 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., storeName=c1cc0773bc61ee89c3c2791bfea1e203/C, priority=13, startTime=1733788617570; duration=0sec 2024-12-09T23:56:58,060 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:56:58,060 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cc0773bc61ee89c3c2791bfea1e203:C 2024-12-09T23:56:58,113 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/2199f0d158e34811b1aa396038ad5642 2024-12-09T23:56:58,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/ec8e8e72066a44c499254bd9fdd54b04 is 50, key is test_row_0/B:col10/1733788616704/Put/seqid=0 2024-12-09T23:56:58,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741895_1071 (size=12301) 2024-12-09T23:56:58,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-09T23:56:58,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:58,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:58,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788678213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:58,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788678213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:58,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:58,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:58,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:58,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:58,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:58,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:58,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:58,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788678213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:58,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:58,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:58,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:58,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:58,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788678512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:58,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:58,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788678512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:58,520 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_2' in c1cc0773bc61ee89c3c2791bfea1e203 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:56:58,520 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_1' in c1cc0773bc61ee89c3c2791bfea1e203 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:56:58,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:58,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:58,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:58,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:58,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:58,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:58,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:58,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:58,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788678523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:58,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788678523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:58,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:58,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:58,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:58,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:58,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:58,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:58,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788678523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:58,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:58,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:58,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:58,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:58,533 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/ec8e8e72066a44c499254bd9fdd54b04 2024-12-09T23:56:58,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/1d7ac4291d624ce281a45aa2a1a34e76 is 50, key is test_row_0/C:col10/1733788616704/Put/seqid=0 2024-12-09T23:56:58,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741896_1072 (size=12301) 2024-12-09T23:56:58,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-09T23:56:58,959 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/1d7ac4291d624ce281a45aa2a1a34e76 2024-12-09T23:56:58,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/2199f0d158e34811b1aa396038ad5642 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/2199f0d158e34811b1aa396038ad5642 2024-12-09T23:56:58,969 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/2199f0d158e34811b1aa396038ad5642, entries=150, sequenceid=371, filesize=12.0 K 2024-12-09T23:56:58,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/ec8e8e72066a44c499254bd9fdd54b04 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/ec8e8e72066a44c499254bd9fdd54b04 2024-12-09T23:56:58,976 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/ec8e8e72066a44c499254bd9fdd54b04, entries=150, sequenceid=371, filesize=12.0 K 2024-12-09T23:56:58,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/1d7ac4291d624ce281a45aa2a1a34e76 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/1d7ac4291d624ce281a45aa2a1a34e76 2024-12-09T23:56:58,983 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/1d7ac4291d624ce281a45aa2a1a34e76, entries=150, sequenceid=371, filesize=12.0 K 2024-12-09T23:56:58,984 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=107.34 KB/109920 for c1cc0773bc61ee89c3c2791bfea1e203 in 1284ms, sequenceid=371, compaction requested=false 2024-12-09T23:56:58,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:56:58,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:58,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-09T23:56:58,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-09T23:56:58,988 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-09T23:56:58,989 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4390 sec 2024-12-09T23:56:58,991 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.4450 sec 2024-12-09T23:56:59,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:56:59,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733788617870, max timestamp=1733788619036Num uniques 859; 2024-12-09T23:56:59,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733788617870, max timestamp=1733788619036Num uniques 859; 2024-12-09T23:56:59,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733788617870, max timestamp=1733788619036Num uniques 859; 2024-12-09T23:56:59,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T23:56:59,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T23:56:59,039 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T23:56:59,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1cc0773bc61ee89c3c2791bfea1e203 3/3 column families, dataSize=33.54 KB heapSize=81.47 KB 2024-12-09T23:56:59,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=A 2024-12-09T23:56:59,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:56:59,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=B 2024-12-09T23:56:59,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:56:59,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=C 2024-12-09T23:56:59,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:56:59,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/5ed8c19b712748a5bb86b9e109eb1448 is 50, key is test_row_0/A:col10/1733788619036/Put/seqid=0 2024-12-09T23:56:59,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741897_1073 (size=12301) 2024-12-09T23:56:59,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/5ed8c19b712748a5bb86b9e109eb1448 2024-12-09T23:56:59,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/10e637f4b15a4d7eb5bb22fb23d113a8 is 50, key is test_row_0/B:col10/1733788619036/Put/seqid=0 2024-12-09T23:56:59,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741898_1074 (size=12301) 2024-12-09T23:56:59,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733788619060, max timestamp=1733788619117Num uniques 859; 2024-12-09T23:56:59,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T23:56:59,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733788619060, max timestamp=1733788619117Num uniques 859; 2024-12-09T23:56:59,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T23:56:59,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733788619060, max timestamp=1733788619117Num uniques 859; 2024-12-09T23:56:59,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T23:56:59,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:59,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788679145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:59,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:59,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:59,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:59,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:59,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788679150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:59,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:59,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:59,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:59,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788679150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:59,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:59,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:59,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:59,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:59,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788679252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:59,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:59,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:59,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:59,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:59,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788679263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:59,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:59,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:59,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:59,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:59,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788679263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:59,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:59,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:59,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:59,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/10e637f4b15a4d7eb5bb22fb23d113a8 2024-12-09T23:56:59,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:59,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788679463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:59,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:59,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:59,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:59,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:59,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788679473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:59,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:59,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:59,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788679473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:59,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:59,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:59,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:59,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/9aeabdedfd744be69105f074affa760c is 50, key is test_row_0/C:col10/1733788619036/Put/seqid=0 2024-12-09T23:56:59,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741899_1075 (size=12301) 2024-12-09T23:56:59,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/9aeabdedfd744be69105f074affa760c 2024-12-09T23:56:59,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/5ed8c19b712748a5bb86b9e109eb1448 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/5ed8c19b712748a5bb86b9e109eb1448 2024-12-09T23:56:59,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/5ed8c19b712748a5bb86b9e109eb1448, entries=150, sequenceid=396, filesize=12.0 K 2024-12-09T23:56:59,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/10e637f4b15a4d7eb5bb22fb23d113a8 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/10e637f4b15a4d7eb5bb22fb23d113a8 2024-12-09T23:56:59,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/10e637f4b15a4d7eb5bb22fb23d113a8, entries=150, sequenceid=396, filesize=12.0 K 2024-12-09T23:56:59,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/9aeabdedfd744be69105f074affa760c as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/9aeabdedfd744be69105f074affa760c 2024-12-09T23:56:59,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/9aeabdedfd744be69105f074affa760c, entries=150, sequenceid=396, filesize=12.0 K 2024-12-09T23:56:59,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~101.34 KB/103776, currentSize=147.60 KB/151140 for c1cc0773bc61ee89c3c2791bfea1e203 in 512ms, sequenceid=396, compaction requested=true 2024-12-09T23:56:59,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:56:59,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cc0773bc61ee89c3c2791bfea1e203:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:56:59,552 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:56:59,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:56:59,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cc0773bc61ee89c3c2791bfea1e203:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:56:59,552 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:56:59,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:56:59,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cc0773bc61ee89c3c2791bfea1e203:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:56:59,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:56:59,553 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:56:59,553 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:56:59,553 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): c1cc0773bc61ee89c3c2791bfea1e203/A is initiating minor compaction (all files) 2024-12-09T23:56:59,553 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): c1cc0773bc61ee89c3c2791bfea1e203/B is initiating minor compaction (all files) 2024-12-09T23:56:59,553 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cc0773bc61ee89c3c2791bfea1e203/A in TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:59,553 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cc0773bc61ee89c3c2791bfea1e203/B in TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:59,553 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/3ebd779e73e5460d8c4c588d44c5a746, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/ec8e8e72066a44c499254bd9fdd54b04, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/10e637f4b15a4d7eb5bb22fb23d113a8] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp, totalSize=36.6 K 2024-12-09T23:56:59,553 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/4d2af723ad5d4c348340f68f6baac4d2, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/2199f0d158e34811b1aa396038ad5642, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/5ed8c19b712748a5bb86b9e109eb1448] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp, totalSize=36.6 K 2024-12-09T23:56:59,554 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ebd779e73e5460d8c4c588d44c5a746, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1733788616353 2024-12-09T23:56:59,554 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d2af723ad5d4c348340f68f6baac4d2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1733788616353 2024-12-09T23:56:59,554 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting ec8e8e72066a44c499254bd9fdd54b04, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733788616700 2024-12-09T23:56:59,554 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2199f0d158e34811b1aa396038ad5642, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733788616700 2024-12-09T23:56:59,555 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ed8c19b712748a5bb86b9e109eb1448, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1733788617878 2024-12-09T23:56:59,555 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 10e637f4b15a4d7eb5bb22fb23d113a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1733788617878 2024-12-09T23:56:59,565 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cc0773bc61ee89c3c2791bfea1e203#A#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:56:59,565 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/cb9cfc69e047401f919135ff4226fa42 is 50, key is test_row_0/A:col10/1733788619036/Put/seqid=0 2024-12-09T23:56:59,570 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cc0773bc61ee89c3c2791bfea1e203#B#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:56:59,570 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/fbd409cff57042789d6d022f1f285fed is 50, key is test_row_0/B:col10/1733788619036/Put/seqid=0 2024-12-09T23:56:59,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741900_1076 (size=13017) 2024-12-09T23:56:59,585 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/cb9cfc69e047401f919135ff4226fa42 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/cb9cfc69e047401f919135ff4226fa42 2024-12-09T23:56:59,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741901_1077 (size=13017) 2024-12-09T23:56:59,599 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1cc0773bc61ee89c3c2791bfea1e203/A of c1cc0773bc61ee89c3c2791bfea1e203 into cb9cfc69e047401f919135ff4226fa42(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:56:59,599 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:56:59,599 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., storeName=c1cc0773bc61ee89c3c2791bfea1e203/A, priority=13, startTime=1733788619552; duration=0sec 2024-12-09T23:56:59,599 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:56:59,599 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cc0773bc61ee89c3c2791bfea1e203:A 2024-12-09T23:56:59,599 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:56:59,601 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:56:59,601 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): c1cc0773bc61ee89c3c2791bfea1e203/C is initiating minor compaction (all files) 2024-12-09T23:56:59,601 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cc0773bc61ee89c3c2791bfea1e203/C in TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:59,601 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/150e3281f14a481fb8885e7224e2bd4f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/1d7ac4291d624ce281a45aa2a1a34e76, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/9aeabdedfd744be69105f074affa760c] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp, totalSize=36.6 K 2024-12-09T23:56:59,602 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 150e3281f14a481fb8885e7224e2bd4f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1733788616353 2024-12-09T23:56:59,602 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d7ac4291d624ce281a45aa2a1a34e76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733788616700 2024-12-09T23:56:59,603 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9aeabdedfd744be69105f074affa760c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1733788617878 2024-12-09T23:56:59,611 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cc0773bc61ee89c3c2791bfea1e203#C#compaction#65 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:56:59,611 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/d67e2984e27b4b878ec4a64a77dd1ad3 is 50, key is test_row_0/C:col10/1733788619036/Put/seqid=0 2024-12-09T23:56:59,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741902_1078 (size=13017) 2024-12-09T23:56:59,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-09T23:56:59,693 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:56:59,693 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:56:59,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-09T23:56:59,695 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:56:59,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T23:56:59,696 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:56:59,696 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:56:59,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:56:59,775 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1cc0773bc61ee89c3c2791bfea1e203 3/3 column families, dataSize=154.31 KB heapSize=436.08 KB 2024-12-09T23:56:59,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=A 2024-12-09T23:56:59,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T23:56:59,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=B 2024-12-09T23:56:59,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T23:56:59,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=C 2024-12-09T23:56:59,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T23:56:59,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/c77cefcbc7bb4c27856960b8890c9896 is 50, key is test_row_0/A:col10/1733788619148/Put/seqid=0 2024-12-09T23:56:59,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741903_1079 (size=14741) 2024-12-09T23:56:59,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:59,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788679784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:59,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:59,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:59,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:59,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:59,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788679788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:59,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:59,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788679788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:59,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:59,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:59,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:59,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:59,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:59,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:59,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T23:56:59,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T23:56:59,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:59,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:56:59,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:56:59,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:59,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:59,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:56:59,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:59,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788679893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:59,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:59,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:59,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:59,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:59,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788679893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:59,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:59,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:59,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:56:59,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:56:59,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788679903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:56:59,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:59,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:56:59,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:00,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T23:57:00,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:00,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:57:00,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:00,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:00,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:00,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:00,006 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/fbd409cff57042789d6d022f1f285fed as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/fbd409cff57042789d6d022f1f285fed 2024-12-09T23:57:00,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T23:57:00,014 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1cc0773bc61ee89c3c2791bfea1e203/B of c1cc0773bc61ee89c3c2791bfea1e203 into fbd409cff57042789d6d022f1f285fed(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:00,014 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:57:00,014 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., storeName=c1cc0773bc61ee89c3c2791bfea1e203/B, priority=13, startTime=1733788619552; duration=0sec 2024-12-09T23:57:00,014 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:00,014 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cc0773bc61ee89c3c2791bfea1e203:B 2024-12-09T23:57:00,029 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/d67e2984e27b4b878ec4a64a77dd1ad3 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/d67e2984e27b4b878ec4a64a77dd1ad3 2024-12-09T23:57:00,034 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1cc0773bc61ee89c3c2791bfea1e203/C of c1cc0773bc61ee89c3c2791bfea1e203 into d67e2984e27b4b878ec4a64a77dd1ad3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:00,034 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:57:00,034 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., storeName=c1cc0773bc61ee89c3c2791bfea1e203/C, priority=13, startTime=1733788619552; duration=0sec 2024-12-09T23:57:00,034 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:00,034 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cc0773bc61ee89c3c2791bfea1e203:C 2024-12-09T23:57:00,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:00,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:00,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788680102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:00,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788680102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:00,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:00,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:00,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:00,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:00,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:00,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:00,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:00,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788680113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:00,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:00,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:00,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:00,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T23:57:00,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:00,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:57:00,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:00,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:00,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:00,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:00,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/c77cefcbc7bb4c27856960b8890c9896 2024-12-09T23:57:00,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/c1a717c08f8b43938ed59fc67a32e0b4 is 50, key is test_row_0/B:col10/1733788619148/Put/seqid=0 2024-12-09T23:57:00,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741904_1080 (size=12301) 2024-12-09T23:57:00,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T23:57:00,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:00,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:57:00,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:00,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:00,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:00,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:00,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T23:57:00,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:00,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:00,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788680412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:00,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788680412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:00,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:00,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:00,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:00,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:00,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:00,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:00,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:00,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788680423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:00,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:00,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:00,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:00,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T23:57:00,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:00,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:57:00,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:00,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:00,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:00,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:00,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:00,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788680533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:00,537 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_2' in c1cc0773bc61ee89c3c2791bfea1e203 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:57:00,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:00,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:00,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:00,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:00,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788680543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:00,545 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_1' in c1cc0773bc61ee89c3c2791bfea1e203 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:57:00,546 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:00,546 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:00,546 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:00,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/c1a717c08f8b43938ed59fc67a32e0b4 2024-12-09T23:57:00,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/b2d7abf442924cad8e32688a1dee6401 is 50, key is test_row_0/C:col10/1733788619148/Put/seqid=0 2024-12-09T23:57:00,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741905_1081 (size=12301) 2024-12-09T23:57:00,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/b2d7abf442924cad8e32688a1dee6401 2024-12-09T23:57:00,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/c77cefcbc7bb4c27856960b8890c9896 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/c77cefcbc7bb4c27856960b8890c9896 2024-12-09T23:57:00,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T23:57:00,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:00,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:57:00,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:00,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:00,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:00,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:00,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/c77cefcbc7bb4c27856960b8890c9896, entries=200, sequenceid=435, filesize=14.4 K 2024-12-09T23:57:00,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/c1a717c08f8b43938ed59fc67a32e0b4 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/c1a717c08f8b43938ed59fc67a32e0b4 2024-12-09T23:57:00,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/c1a717c08f8b43938ed59fc67a32e0b4, entries=150, sequenceid=435, filesize=12.0 K 2024-12-09T23:57:00,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/b2d7abf442924cad8e32688a1dee6401 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/b2d7abf442924cad8e32688a1dee6401 2024-12-09T23:57:00,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/b2d7abf442924cad8e32688a1dee6401, entries=150, sequenceid=435, filesize=12.0 K 2024-12-09T23:57:00,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~436.03 KB/446496, currentSize=26.84 KB/27480 for c1cc0773bc61ee89c3c2791bfea1e203 in 879ms, sequenceid=435, compaction requested=false 2024-12-09T23:57:00,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:57:00,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T23:57:00,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:00,782 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing c1cc0773bc61ee89c3c2791bfea1e203 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-09T23:57:00,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=A 2024-12-09T23:57:00,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:00,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=B 2024-12-09T23:57:00,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:00,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=C 2024-12-09T23:57:00,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:00,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/6684d32bb202407894924b42f51b0681 is 50, key is test_row_0/A:col10/1733788619786/Put/seqid=0 2024-12-09T23:57:00,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741906_1082 (size=9857) 2024-12-09T23:57:00,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T23:57:00,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:57:00,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:57:00,950 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733788620935, max timestamp=1733788620948Num uniques 859; 2024-12-09T23:57:00,950 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T23:57:00,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733788620935, max timestamp=1733788620948Num uniques 859; 2024-12-09T23:57:00,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733788620935, max timestamp=1733788620948Num uniques 859; 2024-12-09T23:57:00,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T23:57:00,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T23:57:00,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:00,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788680981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:00,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9977, totalHeapSize=22912, min timestamp=1733788620974, max timestamp=1733788620981Num uniques 1168; 2024-12-09T23:57:00,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53487, new segment data size=9977, suffix heap size=149440,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=218 2024-12-09T23:57:00,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9970, totalHeapSize=22912, min timestamp=1733788620974, max timestamp=1733788620981Num uniques 1168; 2024-12-09T23:57:00,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=53480, new segment data size=9970, suffix heap size=149440,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=218 2024-12-09T23:57:00,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:00,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733788620974, max timestamp=1733788620981Num uniques 1168; 2024-12-09T23:57:00,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:00,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:00,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=9978, suffix heap size=149440,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=218 2024-12-09T23:57:01,022 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733788621004, max timestamp=1733788621018Num uniques 1077; 2024-12-09T23:57:01,022 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733788621004, max timestamp=1733788621018Num uniques 1077; 2024-12-09T23:57:01,022 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733788621004, max timestamp=1733788621018Num uniques 1077; 2024-12-09T23:57:01,022 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-09T23:57:01,022 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-09T23:57:01,022 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-09T23:57:01,060 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733788621038, max timestamp=1733788621057Num uniques 1086; 2024-12-09T23:57:01,060 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-09T23:57:01,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733788621038, max timestamp=1733788621057Num uniques 1086; 2024-12-09T23:57:01,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-09T23:57:01,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733788621038, max timestamp=1733788621057Num uniques 1086; 2024-12-09T23:57:01,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-09T23:57:01,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1733788621079, max timestamp=1733788621090Num uniques 1095; 2024-12-09T23:57:01,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1733788621079, max timestamp=1733788621090Num uniques 1095; 2024-12-09T23:57:01,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=15800, suffix heap size=141848,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=345 2024-12-09T23:57:01,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1733788621079, max timestamp=1733788621090Num uniques 1095; 2024-12-09T23:57:01,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=15800, suffix heap size=141848,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=345 2024-12-09T23:57:01,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=15800, suffix heap size=141848,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=345 2024-12-09T23:57:01,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:01,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788681119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:01,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:01,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:01,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:01,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:01,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788681122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:01,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:01,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:01,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:01,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:01,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788681123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:01,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:01,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:01,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:01,198 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/6684d32bb202407894924b42f51b0681 2024-12-09T23:57:01,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/94baea67ce184506b5fd6c109b0aba4c is 50, key is test_row_0/B:col10/1733788619786/Put/seqid=0 2024-12-09T23:57:01,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741907_1083 (size=9857) 2024-12-09T23:57:01,219 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/94baea67ce184506b5fd6c109b0aba4c 2024-12-09T23:57:01,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/95d8fa7d58fc442387783f6d36215758 is 50, key is test_row_0/C:col10/1733788619786/Put/seqid=0 2024-12-09T23:57:01,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741908_1084 (size=9857) 2024-12-09T23:57:01,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:01,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:01,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788681234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:01,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788681233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:01,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:01,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788681235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:01,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:01,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:01,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:01,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:01,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:01,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:01,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:01,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:01,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:01,241 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/95d8fa7d58fc442387783f6d36215758 2024-12-09T23:57:01,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/6684d32bb202407894924b42f51b0681 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/6684d32bb202407894924b42f51b0681 2024-12-09T23:57:01,257 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/6684d32bb202407894924b42f51b0681, entries=100, sequenceid=444, filesize=9.6 K 2024-12-09T23:57:01,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/94baea67ce184506b5fd6c109b0aba4c as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/94baea67ce184506b5fd6c109b0aba4c 2024-12-09T23:57:01,263 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/94baea67ce184506b5fd6c109b0aba4c, entries=100, sequenceid=444, filesize=9.6 K 2024-12-09T23:57:01,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/95d8fa7d58fc442387783f6d36215758 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/95d8fa7d58fc442387783f6d36215758 2024-12-09T23:57:01,274 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/95d8fa7d58fc442387783f6d36215758, entries=100, sequenceid=444, filesize=9.6 K 2024-12-09T23:57:01,275 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=161.02 KB/164880 for c1cc0773bc61ee89c3c2791bfea1e203 in 493ms, sequenceid=444, compaction requested=true 2024-12-09T23:57:01,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:57:01,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:01,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-09T23:57:01,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-09T23:57:01,282 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-09T23:57:01,282 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5830 sec 2024-12-09T23:57:01,284 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.5900 sec 2024-12-09T23:57:01,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733788621116, max timestamp=1733788621443Num uniques 1204; 2024-12-09T23:57:01,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=55140, new segment data size=7050, suffix heap size=153184,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1204, new segment cells count=154 2024-12-09T23:57:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:57:01,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733788621116, max timestamp=1733788621443Num uniques 1204; 2024-12-09T23:57:01,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733788621116, max timestamp=1733788621443Num uniques 1204; 2024-12-09T23:57:01,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=55140, new segment data size=7050, suffix heap size=153184,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1204, new segment cells count=154 2024-12-09T23:57:01,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=55140, new segment data size=7050, suffix heap size=153184,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1204, new segment cells count=154 2024-12-09T23:57:01,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1cc0773bc61ee89c3c2791bfea1e203 3/3 column families, dataSize=73.80 KB heapSize=200.42 KB 2024-12-09T23:57:01,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=A 2024-12-09T23:57:01,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:57:01,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=B 2024-12-09T23:57:01,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:57:01,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=C 2024-12-09T23:57:01,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:57:01,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/e89da75ab5d44e06bd24a1d45d076457 is 50, key is test_row_0/A:col10/1733788621116/Put/seqid=0 2024-12-09T23:57:01,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741909_1085 (size=14741) 2024-12-09T23:57:01,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=551 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/e89da75ab5d44e06bd24a1d45d076457 2024-12-09T23:57:01,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/d345503003744e3b9591b9b43ba15502 is 50, key is test_row_0/B:col10/1733788621116/Put/seqid=0 2024-12-09T23:57:01,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741910_1086 (size=12301) 2024-12-09T23:57:01,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733788621459, max timestamp=1733788621483Num uniques 859; 2024-12-09T23:57:01,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T23:57:01,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733788621459, max timestamp=1733788621483Num uniques 859; 2024-12-09T23:57:01,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T23:57:01,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733788621459, max timestamp=1733788621483Num uniques 859; 2024-12-09T23:57:01,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T23:57:01,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733788621528, max timestamp=1733788621584Num uniques 1118; 2024-12-09T23:57:01,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-12-09T23:57:01,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733788621528, max timestamp=1733788621584Num uniques 1009; 2024-12-09T23:57:01,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=9978, suffix heap size=132904,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=218 2024-12-09T23:57:01,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733788621528, max timestamp=1733788621584Num uniques 1009; 2024-12-09T23:57:01,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=9978, suffix heap size=132904,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=218 2024-12-09T23:57:01,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=427, cellSize=19552, totalHeapSize=44648, min timestamp=1733788621607, max timestamp=1733788621622Num uniques 1077; 2024-12-09T23:57:01,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=427, cellSize=19552, totalHeapSize=44648, min timestamp=1733788621607, max timestamp=1733788621622Num uniques 1077; 2024-12-09T23:57:01,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=19552, suffix heap size=139976,new segment heap size=44648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=427 2024-12-09T23:57:01,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=427, cellSize=19552, totalHeapSize=44648, min timestamp=1733788621607, max timestamp=1733788621622Num uniques 1077; 2024-12-09T23:57:01,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=19552, suffix heap size=139976,new segment heap size=44648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=427 2024-12-09T23:57:01,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=19552, suffix heap size=139976,new segment heap size=44648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=427 2024-12-09T23:57:01,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788681657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:01,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:01,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788681658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:01,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:01,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788681665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:01,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:01,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:01,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:01,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:01,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:01,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:01,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:01,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:01,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:01,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:01,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:01,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788681773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:01,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788681773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:01,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:01,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:01,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:01,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:01,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:01,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:01,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:01,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788681774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:01,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:01,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:01,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:01,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T23:57:01,843 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:57:01,844 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:57:01,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-09T23:57:01,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T23:57:01,846 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:57:01,847 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:57:01,847 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:57:01,884 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=551 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/d345503003744e3b9591b9b43ba15502 2024-12-09T23:57:01,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/3c822ca84e4d4272b3bbbf0e13be22e2 is 50, key is test_row_0/C:col10/1733788621116/Put/seqid=0 2024-12-09T23:57:01,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741911_1087 (size=12301) 2024-12-09T23:57:01,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=551 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/3c822ca84e4d4272b3bbbf0e13be22e2 2024-12-09T23:57:01,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/e89da75ab5d44e06bd24a1d45d076457 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/e89da75ab5d44e06bd24a1d45d076457 2024-12-09T23:57:01,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/e89da75ab5d44e06bd24a1d45d076457, entries=200, sequenceid=551, filesize=14.4 K 2024-12-09T23:57:01,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/d345503003744e3b9591b9b43ba15502 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/d345503003744e3b9591b9b43ba15502 2024-12-09T23:57:01,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/d345503003744e3b9591b9b43ba15502, entries=150, sequenceid=551, filesize=12.0 K 2024-12-09T23:57:01,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/3c822ca84e4d4272b3bbbf0e13be22e2 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/3c822ca84e4d4272b3bbbf0e13be22e2 2024-12-09T23:57:01,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/3c822ca84e4d4272b3bbbf0e13be22e2, entries=150, sequenceid=551, filesize=12.0 K 2024-12-09T23:57:01,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.66 KB/68256, currentSize=167.72 KB/171750 for c1cc0773bc61ee89c3c2791bfea1e203 in 484ms, sequenceid=551, compaction requested=true 2024-12-09T23:57:01,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:57:01,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cc0773bc61ee89c3c2791bfea1e203:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:57:01,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:01,932 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T23:57:01,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cc0773bc61ee89c3c2791bfea1e203:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:57:01,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:01,932 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T23:57:01,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cc0773bc61ee89c3c2791bfea1e203:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:57:01,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:57:01,933 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52356 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T23:57:01,934 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): c1cc0773bc61ee89c3c2791bfea1e203/A is initiating minor compaction (all files) 2024-12-09T23:57:01,934 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cc0773bc61ee89c3c2791bfea1e203/A in TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:01,934 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/cb9cfc69e047401f919135ff4226fa42, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/c77cefcbc7bb4c27856960b8890c9896, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/6684d32bb202407894924b42f51b0681, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/e89da75ab5d44e06bd24a1d45d076457] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp, totalSize=51.1 K 2024-12-09T23:57:01,934 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T23:57:01,934 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): c1cc0773bc61ee89c3c2791bfea1e203/B is initiating minor compaction (all files) 2024-12-09T23:57:01,934 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cc0773bc61ee89c3c2791bfea1e203/B in TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:01,935 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/fbd409cff57042789d6d022f1f285fed, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/c1a717c08f8b43938ed59fc67a32e0b4, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/94baea67ce184506b5fd6c109b0aba4c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/d345503003744e3b9591b9b43ba15502] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp, totalSize=46.4 K 2024-12-09T23:57:01,935 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb9cfc69e047401f919135ff4226fa42, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1733788617878 2024-12-09T23:57:01,935 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting fbd409cff57042789d6d022f1f285fed, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1733788617878 2024-12-09T23:57:01,935 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting c77cefcbc7bb4c27856960b8890c9896, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1733788619143 2024-12-09T23:57:01,936 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting c1a717c08f8b43938ed59fc67a32e0b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1733788619148 2024-12-09T23:57:01,936 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6684d32bb202407894924b42f51b0681, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1733788619786 2024-12-09T23:57:01,936 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 94baea67ce184506b5fd6c109b0aba4c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1733788619786 2024-12-09T23:57:01,936 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting e89da75ab5d44e06bd24a1d45d076457, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=551, earliestPutTs=1733788621116 2024-12-09T23:57:01,936 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting d345503003744e3b9591b9b43ba15502, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=551, earliestPutTs=1733788621116 2024-12-09T23:57:01,946 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cc0773bc61ee89c3c2791bfea1e203#B#compaction#75 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:57:01,947 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/ee2be71de7d548849bf265e97ff66b39 is 50, key is test_row_0/B:col10/1733788621116/Put/seqid=0 2024-12-09T23:57:01,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T23:57:01,958 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cc0773bc61ee89c3c2791bfea1e203#A#compaction#76 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:57:01,958 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/7267baa1e8d8443b97ad5d6f7dfcefb9 is 50, key is test_row_0/A:col10/1733788621116/Put/seqid=0 2024-12-09T23:57:01,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741912_1088 (size=13153) 2024-12-09T23:57:01,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741913_1089 (size=13153) 2024-12-09T23:57:01,984 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/7267baa1e8d8443b97ad5d6f7dfcefb9 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/7267baa1e8d8443b97ad5d6f7dfcefb9 2024-12-09T23:57:01,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1cc0773bc61ee89c3c2791bfea1e203 3/3 column families, dataSize=181.14 KB heapSize=500.02 KB 2024-12-09T23:57:01,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=A 2024-12-09T23:57:01,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T23:57:01,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=B 2024-12-09T23:57:01,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T23:57:01,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=C 2024-12-09T23:57:01,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T23:57:01,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:57:01,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:01,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788681989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:01,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:01,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:01,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:01,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:01,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788681990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:01,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:01,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788681991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:01,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:01,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:01,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:01,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:01,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:01,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:02,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-09T23:57:02,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:02,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:57:02,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:02,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:02,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:02,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:02,003 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1cc0773bc61ee89c3c2791bfea1e203/A of c1cc0773bc61ee89c3c2791bfea1e203 into 7267baa1e8d8443b97ad5d6f7dfcefb9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:02,003 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:57:02,003 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., storeName=c1cc0773bc61ee89c3c2791bfea1e203/A, priority=12, startTime=1733788621932; duration=0sec 2024-12-09T23:57:02,003 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:57:02,003 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cc0773bc61ee89c3c2791bfea1e203:A 2024-12-09T23:57:02,003 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T23:57:02,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/350c564b6de64730b6471bd90eea23cd is 50, key is test_row_0/A:col10/1733788621985/Put/seqid=0 2024-12-09T23:57:02,006 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T23:57:02,006 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): c1cc0773bc61ee89c3c2791bfea1e203/C is initiating minor compaction (all files) 2024-12-09T23:57:02,006 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cc0773bc61ee89c3c2791bfea1e203/C in TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:02,006 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/d67e2984e27b4b878ec4a64a77dd1ad3, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/b2d7abf442924cad8e32688a1dee6401, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/95d8fa7d58fc442387783f6d36215758, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/3c822ca84e4d4272b3bbbf0e13be22e2] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp, totalSize=46.4 K 2024-12-09T23:57:02,007 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting d67e2984e27b4b878ec4a64a77dd1ad3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1733788617878 2024-12-09T23:57:02,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741914_1090 (size=12301) 2024-12-09T23:57:02,011 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2d7abf442924cad8e32688a1dee6401, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1733788619148 2024-12-09T23:57:02,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=624 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/350c564b6de64730b6471bd90eea23cd 2024-12-09T23:57:02,013 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95d8fa7d58fc442387783f6d36215758, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1733788619786 2024-12-09T23:57:02,014 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c822ca84e4d4272b3bbbf0e13be22e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=551, earliestPutTs=1733788621116 2024-12-09T23:57:02,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/1b5ddeb6a364403daa70cad67cb9c7c1 is 50, key is test_row_0/B:col10/1733788621985/Put/seqid=0 2024-12-09T23:57:02,026 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cc0773bc61ee89c3c2791bfea1e203#C#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:57:02,027 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/670be54c72754068aba66da9f58dafb9 is 50, key is test_row_0/C:col10/1733788621116/Put/seqid=0 2024-12-09T23:57:02,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741915_1091 (size=12301) 2024-12-09T23:57:02,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=624 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/1b5ddeb6a364403daa70cad67cb9c7c1 2024-12-09T23:57:02,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741916_1092 (size=13153) 2024-12-09T23:57:02,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/2ad8aac4fc2d4f3d9070e9eabdab07be is 50, key is test_row_0/C:col10/1733788621985/Put/seqid=0 2024-12-09T23:57:02,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741917_1093 (size=12301) 2024-12-09T23:57:02,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:02,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788682103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:02,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:02,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788682103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:02,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:02,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788682103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:02,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:02,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:02,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:02,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-09T23:57:02,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:02,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:57:02,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:02,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:02,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:02,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:02,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T23:57:02,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-09T23:57:02,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:02,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:57:02,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:02,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:02,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:02,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:02,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:02,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788682313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:02,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:02,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:02,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:02,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:02,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788682313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:02,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:02,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788682313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:02,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:02,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:02,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:02,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:02,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:02,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:02,372 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/ee2be71de7d548849bf265e97ff66b39 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/ee2be71de7d548849bf265e97ff66b39 2024-12-09T23:57:02,378 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1cc0773bc61ee89c3c2791bfea1e203/B of c1cc0773bc61ee89c3c2791bfea1e203 into ee2be71de7d548849bf265e97ff66b39(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:02,378 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:57:02,378 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., storeName=c1cc0773bc61ee89c3c2791bfea1e203/B, priority=12, startTime=1733788621932; duration=0sec 2024-12-09T23:57:02,378 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:02,378 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cc0773bc61ee89c3c2791bfea1e203:B 2024-12-09T23:57:02,448 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/670be54c72754068aba66da9f58dafb9 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/670be54c72754068aba66da9f58dafb9 2024-12-09T23:57:02,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=624 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/2ad8aac4fc2d4f3d9070e9eabdab07be 2024-12-09T23:57:02,454 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1cc0773bc61ee89c3c2791bfea1e203/C of c1cc0773bc61ee89c3c2791bfea1e203 into 670be54c72754068aba66da9f58dafb9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:02,454 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:57:02,454 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., storeName=c1cc0773bc61ee89c3c2791bfea1e203/C, priority=12, startTime=1733788621932; duration=0sec 2024-12-09T23:57:02,454 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:02,454 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cc0773bc61ee89c3c2791bfea1e203:C 2024-12-09T23:57:02,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/350c564b6de64730b6471bd90eea23cd as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/350c564b6de64730b6471bd90eea23cd 2024-12-09T23:57:02,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-09T23:57:02,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:02,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:57:02,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:02,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:02,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:02,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/350c564b6de64730b6471bd90eea23cd, entries=150, sequenceid=624, filesize=12.0 K 2024-12-09T23:57:02,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:02,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/1b5ddeb6a364403daa70cad67cb9c7c1 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/1b5ddeb6a364403daa70cad67cb9c7c1 2024-12-09T23:57:02,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T23:57:02,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/1b5ddeb6a364403daa70cad67cb9c7c1, entries=150, sequenceid=624, filesize=12.0 K 2024-12-09T23:57:02,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/2ad8aac4fc2d4f3d9070e9eabdab07be as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/2ad8aac4fc2d4f3d9070e9eabdab07be 2024-12-09T23:57:02,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/2ad8aac4fc2d4f3d9070e9eabdab07be, entries=150, sequenceid=624, filesize=12.0 K 2024-12-09T23:57:02,483 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~499.97 KB/511968, currentSize=6.71 KB/6870 for c1cc0773bc61ee89c3c2791bfea1e203 in 497ms, sequenceid=624, compaction requested=false 2024-12-09T23:57:02,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:57:02,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-09T23:57:02,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:02,622 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing c1cc0773bc61ee89c3c2791bfea1e203 3/3 column families, dataSize=6.71 KB heapSize=20.67 KB 2024-12-09T23:57:02,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=A 2024-12-09T23:57:02,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:02,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=B 2024-12-09T23:57:02,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:02,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=C 2024-12-09T23:57:02,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:02,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/5a4ebebe99384c8d9a41459cc8b3cbc4 is 50, key is test_row_2/A:col10/1733788621988/Put/seqid=0 2024-12-09T23:57:02,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:57:02,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:57:02,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741918_1094 (size=7415) 2024-12-09T23:57:02,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733788622634, max timestamp=1733788622672Num uniques 859; 2024-12-09T23:57:02,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733788622634, max timestamp=1733788622672Num uniques 859; 2024-12-09T23:57:02,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T23:57:02,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T23:57:02,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733788622641, max timestamp=1733788622672Num uniques 859; 2024-12-09T23:57:02,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T23:57:02,707 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733788622677, max timestamp=1733788622703Num uniques 1018; 2024-12-09T23:57:02,707 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-09T23:57:02,707 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733788622677, max timestamp=1733788622703Num uniques 1168; 2024-12-09T23:57:02,707 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733788622677, max timestamp=1733788622703Num uniques 1168; 2024-12-09T23:57:02,707 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=9978, suffix heap size=149440,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=218 2024-12-09T23:57:02,709 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=9978, suffix heap size=149440,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=218 2024-12-09T23:57:02,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1733788622718, max timestamp=1733788622736Num uniques 1077; 2024-12-09T23:57:02,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=14972, suffix heap size=139976,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=327 2024-12-09T23:57:02,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1733788622718, max timestamp=1733788622736Num uniques 1077; 2024-12-09T23:57:02,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1733788622718, max timestamp=1733788622736Num uniques 1077; 2024-12-09T23:57:02,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=14972, suffix heap size=139976,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=327 2024-12-09T23:57:02,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=14972, suffix heap size=139976,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=327 2024-12-09T23:57:02,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733788622756, max timestamp=1733788622768Num uniques 1186; 2024-12-09T23:57:02,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13086, totalHeapSize=29984, min timestamp=1733788622756, max timestamp=1733788622768Num uniques 1186; 2024-12-09T23:57:02,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13086, totalHeapSize=29984, min timestamp=1733788622756, max timestamp=1733788622768Num uniques 1186; 2024-12-09T23:57:02,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=13096, suffix heap size=151312,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=286 2024-12-09T23:57:02,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=54306, new segment data size=13086, suffix heap size=151312,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=286 2024-12-09T23:57:02,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=54306, new segment data size=13086, suffix heap size=151312,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=286 2024-12-09T23:57:02,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1733788622779, max timestamp=1733788622802Num uniques 1145; 2024-12-09T23:57:02,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=15800, suffix heap size=147048,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=345 2024-12-09T23:57:02,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1733788622779, max timestamp=1733788622802Num uniques 1145; 2024-12-09T23:57:02,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=15800, suffix heap size=147048,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=345 2024-12-09T23:57:02,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1733788622779, max timestamp=1733788622802Num uniques 1145; 2024-12-09T23:57:02,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=15800, suffix heap size=147048,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=345 2024-12-09T23:57:02,845 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733788622826, max timestamp=1733788622841Num uniques 1204; 2024-12-09T23:57:02,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=55140, new segment data size=9340, suffix heap size=153184,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1204, new segment cells count=204 2024-12-09T23:57:02,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733788622826, max timestamp=1733788622841Num uniques 1204; 2024-12-09T23:57:02,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=55140, new segment data size=9340, suffix heap size=153184,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1204, new segment cells count=204 2024-12-09T23:57:02,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733788622826, max timestamp=1733788622841Num uniques 1204; 2024-12-09T23:57:02,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=55140, new segment data size=7050, suffix heap size=153184,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1204, new segment cells count=154 2024-12-09T23:57:02,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733788622905, max timestamp=1733788622909Num uniques 1063; 2024-12-09T23:57:02,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-12-09T23:57:02,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733788622905, max timestamp=1733788622909Num uniques 1063; 2024-12-09T23:57:02,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-12-09T23:57:02,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733788622905, max timestamp=1733788622909Num uniques 1013; 2024-12-09T23:57:02,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-09T23:57:02,950 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=372, cellSize=17032, totalHeapSize=38928, min timestamp=1733788622919, max timestamp=1733788622947Num uniques 1022; 2024-12-09T23:57:02,950 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=17032, suffix heap size=134256,new segment heap size=38928 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=372 2024-12-09T23:57:02,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=372, cellSize=17032, totalHeapSize=38928, min timestamp=1733788622919, max timestamp=1733788622947Num uniques 1022; 2024-12-09T23:57:02,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=17032, suffix heap size=134256,new segment heap size=38928 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=372 2024-12-09T23:57:02,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=372, cellSize=17032, totalHeapSize=38928, min timestamp=1733788622919, max timestamp=1733788622947Num uniques 1022; 2024-12-09T23:57:02,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=17032, suffix heap size=134256,new segment heap size=38928 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=372 2024-12-09T23:57:02,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T23:57:02,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733788622974, max timestamp=1733788622991Num uniques 1231; 2024-12-09T23:57:02,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=56376, new segment data size=10576, suffix heap size=155992,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1231, new segment cells count=231 2024-12-09T23:57:02,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733788622974, max timestamp=1733788622991Num uniques 1231; 2024-12-09T23:57:02,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=56376, new segment data size=10576, suffix heap size=155992,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1231, new segment cells count=231 2024-12-09T23:57:02,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733788622974, max timestamp=1733788622991Num uniques 1231; 2024-12-09T23:57:02,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=56376, new segment data size=10576, suffix heap size=155992,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1231, new segment cells count=231 2024-12-09T23:57:03,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733788623002, max timestamp=1733788623020Num uniques 1090; 2024-12-09T23:57:03,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-12-09T23:57:03,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733788623002, max timestamp=1733788623020Num uniques 1090; 2024-12-09T23:57:03,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-12-09T23:57:03,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733788623002, max timestamp=1733788623020Num uniques 1090; 2024-12-09T23:57:03,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-12-09T23:57:03,039 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=631 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/5a4ebebe99384c8d9a41459cc8b3cbc4 2024-12-09T23:57:03,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/af4166b40bd440dea035d4a356581f7d is 50, key is test_row_2/B:col10/1733788621988/Put/seqid=0 2024-12-09T23:57:03,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741919_1095 (size=7415) 2024-12-09T23:57:03,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1733788623027, max timestamp=1733788623059Num uniques 1049; 2024-12-09T23:57:03,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1733788623027, max timestamp=1733788623059Num uniques 1049; 2024-12-09T23:57:03,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-12-09T23:57:03,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-12-09T23:57:03,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9104, totalHeapSize=20936, min timestamp=1733788623027, max timestamp=1733788623059Num uniques 1049; 2024-12-09T23:57:03,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48034, new segment data size=9104, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-12-09T23:57:03,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=308, cellSize=14100, totalHeapSize=32272, min timestamp=1733788623079, max timestamp=1733788623092Num uniques 1058; 2024-12-09T23:57:03,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=14100, suffix heap size=138000,new segment heap size=32272 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=308 2024-12-09T23:57:03,096 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=308, cellSize=14100, totalHeapSize=32272, min timestamp=1733788623079, max timestamp=1733788623092Num uniques 1058; 2024-12-09T23:57:03,096 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=308, cellSize=14100, totalHeapSize=32272, min timestamp=1733788623079, max timestamp=1733788623092Num uniques 1058; 2024-12-09T23:57:03,096 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=14100, suffix heap size=138000,new segment heap size=32272 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=308 2024-12-09T23:57:03,096 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=14100, suffix heap size=138000,new segment heap size=32272 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=308 2024-12-09T23:57:03,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=317, cellSize=14512, totalHeapSize=33208, min timestamp=1733788623108, max timestamp=1733788623126Num uniques 1167; 2024-12-09T23:57:03,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=53442, new segment data size=14512, suffix heap size=149336,new segment heap size=33208 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1167, new segment cells count=317 2024-12-09T23:57:03,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=317, cellSize=14512, totalHeapSize=33208, min timestamp=1733788623108, max timestamp=1733788623126Num uniques 1167; 2024-12-09T23:57:03,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=53442, new segment data size=14512, suffix heap size=149336,new segment heap size=33208 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1167, new segment cells count=317 2024-12-09T23:57:03,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=317, cellSize=14512, totalHeapSize=33208, min timestamp=1733788623108, max timestamp=1733788623126Num uniques 1167; 2024-12-09T23:57:03,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=53442, new segment data size=14512, suffix heap size=149336,new segment heap size=33208 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1167, new segment cells count=317 2024-12-09T23:57:03,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=426, cellSize=19506, totalHeapSize=44544, min timestamp=1733788623136, max timestamp=1733788623150Num uniques 1176; 2024-12-09T23:57:03,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=53856, new segment data size=19506, suffix heap size=150272,new segment heap size=44544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1176, new segment cells count=426 2024-12-09T23:57:03,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=426, cellSize=19506, totalHeapSize=44544, min timestamp=1733788623136, max timestamp=1733788623150Num uniques 1176; 2024-12-09T23:57:03,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=53856, new segment data size=19506, suffix heap size=150272,new segment heap size=44544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1176, new segment cells count=426 2024-12-09T23:57:03,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=426, cellSize=19506, totalHeapSize=44544, min timestamp=1733788623136, max timestamp=1733788623150Num uniques 1176; 2024-12-09T23:57:03,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=53856, new segment data size=19506, suffix heap size=150272,new segment heap size=44544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1176, new segment cells count=426 2024-12-09T23:57:03,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1733788623164, max timestamp=1733788623174Num uniques 1285; 2024-12-09T23:57:03,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=58850, new segment data size=10760, suffix heap size=161608,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1285, new segment cells count=235 2024-12-09T23:57:03,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1733788623164, max timestamp=1733788623174Num uniques 1285; 2024-12-09T23:57:03,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=58850, new segment data size=10760, suffix heap size=161608,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1285, new segment cells count=235 2024-12-09T23:57:03,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1733788623164, max timestamp=1733788623174Num uniques 1285; 2024-12-09T23:57:03,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=58850, new segment data size=10760, suffix heap size=161608,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1285, new segment cells count=235 2024-12-09T23:57:03,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=294, cellSize=13464, totalHeapSize=30816, min timestamp=1733788623183, max timestamp=1733788623195Num uniques 1094; 2024-12-09T23:57:03,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=13464, suffix heap size=141744,new segment heap size=30816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=294 2024-12-09T23:57:03,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=294, cellSize=13464, totalHeapSize=30816, min timestamp=1733788623183, max timestamp=1733788623195Num uniques 1094; 2024-12-09T23:57:03,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=13464, suffix heap size=141744,new segment heap size=30816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=294 2024-12-09T23:57:03,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=294, cellSize=13464, totalHeapSize=30816, min timestamp=1733788623183, max timestamp=1733788623195Num uniques 1094; 2024-12-09T23:57:03,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=13464, suffix heap size=141744,new segment heap size=30816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=294 2024-12-09T23:57:03,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=303, cellSize=13875, totalHeapSize=31752, min timestamp=1733788623211, max timestamp=1733788623223Num uniques 1153; 2024-12-09T23:57:03,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=303, cellSize=13875, totalHeapSize=31752, min timestamp=1733788623211, max timestamp=1733788623223Num uniques 1153; 2024-12-09T23:57:03,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=303, cellSize=13875, totalHeapSize=31752, min timestamp=1733788623211, max timestamp=1733788623223Num uniques 1153; 2024-12-09T23:57:03,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52805, new segment data size=13875, suffix heap size=147880,new segment heap size=31752 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1153, new segment cells count=303 2024-12-09T23:57:03,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52805, new segment data size=13875, suffix heap size=147880,new segment heap size=31752 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1153, new segment cells count=303 2024-12-09T23:57:03,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52805, new segment data size=13875, suffix heap size=147880,new segment heap size=31752 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1153, new segment cells count=303 2024-12-09T23:57:03,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=312, cellSize=14282, totalHeapSize=32688, min timestamp=1733788623228, max timestamp=1733788623263Num uniques 1162; 2024-12-09T23:57:03,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=312, cellSize=14282, totalHeapSize=32688, min timestamp=1733788623228, max timestamp=1733788623263Num uniques 1162; 2024-12-09T23:57:03,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=53212, new segment data size=14282, suffix heap size=148816,new segment heap size=32688 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1162, new segment cells count=312 2024-12-09T23:57:03,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=53212, new segment data size=14282, suffix heap size=148816,new segment heap size=32688 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1162, new segment cells count=312 2024-12-09T23:57:03,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=312, cellSize=14282, totalHeapSize=32688, min timestamp=1733788623228, max timestamp=1733788623263Num uniques 1162; 2024-12-09T23:57:03,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=53212, new segment data size=14282, suffix heap size=148816,new segment heap size=32688 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1162, new segment cells count=312 2024-12-09T23:57:03,291 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=371, cellSize=16986, totalHeapSize=38824, min timestamp=1733788623277, max timestamp=1733788623289Num uniques 1171; 2024-12-09T23:57:03,291 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=53626, new segment data size=16986, suffix heap size=149752,new segment heap size=38824 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1171, new segment cells count=371 2024-12-09T23:57:03,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=371, cellSize=16986, totalHeapSize=38824, min timestamp=1733788623277, max timestamp=1733788623289Num uniques 1171; 2024-12-09T23:57:03,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=53626, new segment data size=16986, suffix heap size=149752,new segment heap size=38824 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1171, new segment cells count=371 2024-12-09T23:57:03,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=371, cellSize=16986, totalHeapSize=38824, min timestamp=1733788623277, max timestamp=1733788623289Num uniques 1171; 2024-12-09T23:57:03,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53626, new segment data size=16986, suffix heap size=149752,new segment heap size=38824 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1171, new segment cells count=371 2024-12-09T23:57:03,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=280, cellSize=12820, totalHeapSize=29360, min timestamp=1733788623303, max timestamp=1733788623309Num uniques 1230; 2024-12-09T23:57:03,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=280, cellSize=12820, totalHeapSize=29360, min timestamp=1733788623303, max timestamp=1733788623309Num uniques 1230; 2024-12-09T23:57:03,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=56330, new segment data size=12820, suffix heap size=155888,new segment heap size=29360 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1230, new segment cells count=280 2024-12-09T23:57:03,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=56330, new segment data size=12820, suffix heap size=155888,new segment heap size=29360 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1230, new segment cells count=280 2024-12-09T23:57:03,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=280, cellSize=12820, totalHeapSize=29360, min timestamp=1733788623303, max timestamp=1733788623309Num uniques 1230; 2024-12-09T23:57:03,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:03,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=56330, new segment data size=12820, suffix heap size=155888,new segment heap size=29360 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1230, new segment cells count=280 2024-12-09T23:57:03,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 332 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788683311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:03,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:03,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:03,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:03,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=239, cellSize=10944, totalHeapSize=25096, min timestamp=1733788623334, max timestamp=1733788623344Num uniques 1139; 2024-12-09T23:57:03,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52164, new segment data size=10944, suffix heap size=146424,new segment heap size=25096 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1139, new segment cells count=239 2024-12-09T23:57:03,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=239, cellSize=10944, totalHeapSize=25096, min timestamp=1733788623334, max timestamp=1733788623344Num uniques 1139; 2024-12-09T23:57:03,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52164, new segment data size=10944, suffix heap size=146424,new segment heap size=25096 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1139, new segment cells count=239 2024-12-09T23:57:03,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=239, cellSize=10944, totalHeapSize=25096, min timestamp=1733788623334, max timestamp=1733788623344Num uniques 1139; 2024-12-09T23:57:03,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52164, new segment data size=10944, suffix heap size=146424,new segment heap size=25096 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1139, new segment cells count=239 2024-12-09T23:57:03,373 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=198, cellSize=9061, totalHeapSize=20832, min timestamp=1733788623361, max timestamp=1733788623370Num uniques 1098; 2024-12-09T23:57:03,373 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50281, new segment data size=9061, suffix heap size=142160,new segment heap size=20832 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1098, new segment cells count=198 2024-12-09T23:57:03,373 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=198, cellSize=9058, totalHeapSize=20832, min timestamp=1733788623361, max timestamp=1733788623370Num uniques 1098; 2024-12-09T23:57:03,373 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50278, new segment data size=9058, suffix heap size=142160,new segment heap size=20832 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1098, new segment cells count=198 2024-12-09T23:57:03,373 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=198, cellSize=9058, totalHeapSize=20832, min timestamp=1733788623361, max timestamp=1733788623370Num uniques 1098; 2024-12-09T23:57:03,373 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50278, new segment data size=9058, suffix heap size=142160,new segment heap size=20832 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1098, new segment cells count=198 2024-12-09T23:57:03,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=207, cellSize=9475, totalHeapSize=21768, min timestamp=1733788623381, max timestamp=1733788623403Num uniques 1057; 2024-12-09T23:57:03,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48405, new segment data size=9475, suffix heap size=137896,new segment heap size=21768 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1057, new segment cells count=207 2024-12-09T23:57:03,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=207, cellSize=9475, totalHeapSize=21768, min timestamp=1733788623381, max timestamp=1733788623403Num uniques 1057; 2024-12-09T23:57:03,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48405, new segment data size=9475, suffix heap size=137896,new segment heap size=21768 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1057, new segment cells count=207 2024-12-09T23:57:03,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=207, cellSize=9475, totalHeapSize=21768, min timestamp=1733788623381, max timestamp=1733788623403Num uniques 1057; 2024-12-09T23:57:03,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48405, new segment data size=9475, suffix heap size=137896,new segment heap size=21768 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1057, new segment cells count=207 2024-12-09T23:57:03,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=216, cellSize=9886, totalHeapSize=22704, min timestamp=1733788623418, max timestamp=1733788623428Num uniques 1066; 2024-12-09T23:57:03,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=266, cellSize=12176, totalHeapSize=27904, min timestamp=1733788623418, max timestamp=1733788623428Num uniques 1066; 2024-12-09T23:57:03,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48816, new segment data size=9886, suffix heap size=138832,new segment heap size=22704 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1066, new segment cells count=216 2024-12-09T23:57:03,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733788623418, max timestamp=1733788623428Num uniques 1066; 2024-12-09T23:57:03,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48816, new segment data size=12176, suffix heap size=138832,new segment heap size=27904 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1066, new segment cells count=266 2024-12-09T23:57:03,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48816, new segment data size=6870, suffix heap size=138832,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1066, new segment cells count=150 2024-12-09T23:57:03,462 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=631 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/af4166b40bd440dea035d4a356581f7d 2024-12-09T23:57:03,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/27d9acb9c4ad47cfaa8e271712dec620 is 50, key is test_row_2/C:col10/1733788621988/Put/seqid=0 2024-12-09T23:57:03,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741920_1096 (size=7415) 2024-12-09T23:57:03,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=325, cellSize=14880, totalHeapSize=34040, min timestamp=1733788623457, max timestamp=1733788623477Num uniques 1009; 2024-12-09T23:57:03,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=14880, suffix heap size=132904,new segment heap size=34040 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=325 2024-12-09T23:57:03,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=325, cellSize=14880, totalHeapSize=34040, min timestamp=1733788623457, max timestamp=1733788623477Num uniques 1075; 2024-12-09T23:57:03,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=325, cellSize=14880, totalHeapSize=34040, min timestamp=1733788623457, max timestamp=1733788623477Num uniques 1125; 2024-12-09T23:57:03,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49230, new segment data size=14880, suffix heap size=139768,new segment heap size=34040 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1075, new segment cells count=325 2024-12-09T23:57:03,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51520, new segment data size=14880, suffix heap size=144968,new segment heap size=34040 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1125, new segment cells count=325 2024-12-09T23:57:03,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=234, cellSize=10714, totalHeapSize=24576, min timestamp=1733788623494, max timestamp=1733788623498Num uniques 1184; 2024-12-09T23:57:03,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=54224, new segment data size=10714, suffix heap size=151104,new segment heap size=24576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1184, new segment cells count=234 2024-12-09T23:57:03,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=234, cellSize=10714, totalHeapSize=24576, min timestamp=1733788623494, max timestamp=1733788623498Num uniques 1184; 2024-12-09T23:57:03,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=54224, new segment data size=10714, suffix heap size=151104,new segment heap size=24576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1184, new segment cells count=234 2024-12-09T23:57:03,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=234, cellSize=10714, totalHeapSize=24576, min timestamp=1733788623494, max timestamp=1733788623498Num uniques 1184; 2024-12-09T23:57:03,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=54224, new segment data size=10714, suffix heap size=151104,new segment heap size=24576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1184, new segment cells count=234 2024-12-09T23:57:03,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=293, cellSize=13418, totalHeapSize=30712, min timestamp=1733788623508, max timestamp=1733788623522Num uniques 1093; 2024-12-09T23:57:03,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50058, new segment data size=13418, suffix heap size=141640,new segment heap size=30712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1093, new segment cells count=293 2024-12-09T23:57:03,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=293, cellSize=13418, totalHeapSize=30712, min timestamp=1733788623508, max timestamp=1733788623522Num uniques 1093; 2024-12-09T23:57:03,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50058, new segment data size=13418, suffix heap size=141640,new segment heap size=30712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1093, new segment cells count=293 2024-12-09T23:57:03,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=293, cellSize=13418, totalHeapSize=30712, min timestamp=1733788623508, max timestamp=1733788623522Num uniques 1093; 2024-12-09T23:57:03,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50058, new segment data size=13418, suffix heap size=141640,new segment heap size=30712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1093, new segment cells count=293 2024-12-09T23:57:03,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=252, cellSize=11540, totalHeapSize=26448, min timestamp=1733788623531, max timestamp=1733788623545Num uniques 1152; 2024-12-09T23:57:03,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=252, cellSize=11540, totalHeapSize=26448, min timestamp=1733788623531, max timestamp=1733788623545Num uniques 1152; 2024-12-09T23:57:03,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=252, cellSize=11540, totalHeapSize=26448, min timestamp=1733788623531, max timestamp=1733788623545Num uniques 1152; 2024-12-09T23:57:03,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52760, new segment data size=11540, suffix heap size=147776,new segment heap size=26448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1152, new segment cells count=252 2024-12-09T23:57:03,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52760, new segment data size=11540, suffix heap size=147776,new segment heap size=26448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1152, new segment cells count=252 2024-12-09T23:57:03,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52760, new segment data size=11540, suffix heap size=147776,new segment heap size=26448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1152, new segment cells count=252 2024-12-09T23:57:03,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=161, cellSize=7366, totalHeapSize=16984, min timestamp=1733788623563, max timestamp=1733788623566Num uniques 1111; 2024-12-09T23:57:03,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=161, cellSize=7366, totalHeapSize=16984, min timestamp=1733788623563, max timestamp=1733788623566Num uniques 1111; 2024-12-09T23:57:03,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=161, cellSize=7366, totalHeapSize=16984, min timestamp=1733788623563, max timestamp=1733788623566Num uniques 1111; 2024-12-09T23:57:03,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50876, new segment data size=7366, suffix heap size=143512,new segment heap size=16984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1111, new segment cells count=161 2024-12-09T23:57:03,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50876, new segment data size=7366, suffix heap size=143512,new segment heap size=16984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1111, new segment cells count=161 2024-12-09T23:57:03,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50876, new segment data size=7366, suffix heap size=143512,new segment heap size=16984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1111, new segment cells count=161 2024-12-09T23:57:03,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=270, cellSize=12360, totalHeapSize=28320, min timestamp=1733788623581, max timestamp=1733788623588Num uniques 1020; 2024-12-09T23:57:03,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=270, cellSize=12360, totalHeapSize=28320, min timestamp=1733788623581, max timestamp=1733788623588Num uniques 1020; 2024-12-09T23:57:03,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46710, new segment data size=12360, suffix heap size=134048,new segment heap size=28320 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1020, new segment cells count=270 2024-12-09T23:57:03,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46710, new segment data size=12360, suffix heap size=134048,new segment heap size=28320 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1020, new segment cells count=270 2024-12-09T23:57:03,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=270, cellSize=12360, totalHeapSize=28320, min timestamp=1733788623581, max timestamp=1733788623588Num uniques 1020; 2024-12-09T23:57:03,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46710, new segment data size=12360, suffix heap size=134048,new segment heap size=28320 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1020, new segment cells count=270 2024-12-09T23:57:03,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=229, cellSize=10484, totalHeapSize=24056, min timestamp=1733788623602, max timestamp=1733788623611Num uniques 1129; 2024-12-09T23:57:03,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51704, new segment data size=10484, suffix heap size=145384,new segment heap size=24056 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1129, new segment cells count=229 2024-12-09T23:57:03,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=229, cellSize=10484, totalHeapSize=24056, min timestamp=1733788623602, max timestamp=1733788623611Num uniques 1129; 2024-12-09T23:57:03,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=229, cellSize=10484, totalHeapSize=24056, min timestamp=1733788623602, max timestamp=1733788623611Num uniques 1129; 2024-12-09T23:57:03,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51704, new segment data size=10484, suffix heap size=145384,new segment heap size=24056 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1129, new segment cells count=229 2024-12-09T23:57:03,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51704, new segment data size=10484, suffix heap size=145384,new segment heap size=24056 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1129, new segment cells count=229 2024-12-09T23:57:03,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=338, cellSize=15478, totalHeapSize=35392, min timestamp=1733788623617, max timestamp=1733788623635Num uniques 1088; 2024-12-09T23:57:03,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49828, new segment data size=15478, suffix heap size=141120,new segment heap size=35392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1088, new segment cells count=338 2024-12-09T23:57:03,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=338, cellSize=15478, totalHeapSize=35392, min timestamp=1733788623617, max timestamp=1733788623635Num uniques 1088; 2024-12-09T23:57:03,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=338, cellSize=15478, totalHeapSize=35392, min timestamp=1733788623617, max timestamp=1733788623635Num uniques 1088; 2024-12-09T23:57:03,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49828, new segment data size=15478, suffix heap size=141120,new segment heap size=35392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1088, new segment cells count=338 2024-12-09T23:57:03,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49828, new segment data size=15478, suffix heap size=141120,new segment heap size=35392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1088, new segment cells count=338 2024-12-09T23:57:03,684 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=197, cellSize=9022, totalHeapSize=20728, min timestamp=1733788623650, max timestamp=1733788623682Num uniques 1197; 2024-12-09T23:57:03,684 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=197, cellSize=9022, totalHeapSize=20728, min timestamp=1733788623650, max timestamp=1733788623682Num uniques 1197; 2024-12-09T23:57:03,684 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=54822, new segment data size=9022, suffix heap size=152456,new segment heap size=20728 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1197, new segment cells count=197 2024-12-09T23:57:03,684 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=54822, new segment data size=9022, suffix heap size=152456,new segment heap size=20728 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1197, new segment cells count=197 2024-12-09T23:57:03,685 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=197, cellSize=9022, totalHeapSize=20728, min timestamp=1733788623650, max timestamp=1733788623682Num uniques 1197; 2024-12-09T23:57:03,685 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=54822, new segment data size=9022, suffix heap size=152456,new segment heap size=20728 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1197, new segment cells count=197 2024-12-09T23:57:03,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=256, cellSize=11720, totalHeapSize=26864, min timestamp=1733788623692, max timestamp=1733788623702Num uniques 1056; 2024-12-09T23:57:03,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48360, new segment data size=11720, suffix heap size=137792,new segment heap size=26864 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1056, new segment cells count=256 2024-12-09T23:57:03,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=256, cellSize=11720, totalHeapSize=26864, min timestamp=1733788623692, max timestamp=1733788623702Num uniques 1056; 2024-12-09T23:57:03,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48360, new segment data size=11720, suffix heap size=137792,new segment heap size=26864 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1056, new segment cells count=256 2024-12-09T23:57:03,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=256, cellSize=11720, totalHeapSize=26864, min timestamp=1733788623692, max timestamp=1733788623702Num uniques 1056; 2024-12-09T23:57:03,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48360, new segment data size=11720, suffix heap size=137792,new segment heap size=26864 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1056, new segment cells count=256 2024-12-09T23:57:03,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=265, cellSize=12130, totalHeapSize=27800, min timestamp=1733788623714, max timestamp=1733788623719Num uniques 1115; 2024-12-09T23:57:03,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51060, new segment data size=12130, suffix heap size=143928,new segment heap size=27800 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1115, new segment cells count=265 2024-12-09T23:57:03,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=265, cellSize=12130, totalHeapSize=27800, min timestamp=1733788623714, max timestamp=1733788623719Num uniques 1115; 2024-12-09T23:57:03,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51060, new segment data size=12130, suffix heap size=143928,new segment heap size=27800 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1115, new segment cells count=265 2024-12-09T23:57:03,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=265, cellSize=12130, totalHeapSize=27800, min timestamp=1733788623714, max timestamp=1733788623719Num uniques 1115; 2024-12-09T23:57:03,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51060, new segment data size=12130, suffix heap size=143928,new segment heap size=27800 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1115, new segment cells count=265 2024-12-09T23:57:03,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=174, cellSize=7964, totalHeapSize=18336, min timestamp=1733788623738, max timestamp=1733788623746Num uniques 1124; 2024-12-09T23:57:03,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=174, cellSize=7964, totalHeapSize=18336, min timestamp=1733788623738, max timestamp=1733788623746Num uniques 1124; 2024-12-09T23:57:03,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51474, new segment data size=7964, suffix heap size=144864,new segment heap size=18336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1124, new segment cells count=174 2024-12-09T23:57:03,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51474, new segment data size=7964, suffix heap size=144864,new segment heap size=18336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1124, new segment cells count=174 2024-12-09T23:57:03,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=174, cellSize=7964, totalHeapSize=18336, min timestamp=1733788623738, max timestamp=1733788623746Num uniques 1124; 2024-12-09T23:57:03,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51474, new segment data size=7964, suffix heap size=144864,new segment heap size=18336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1124, new segment cells count=174 2024-12-09T23:57:03,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=283, cellSize=12958, totalHeapSize=29672, min timestamp=1733788623764, max timestamp=1733788623795Num uniques 1033; 2024-12-09T23:57:03,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=283, cellSize=12958, totalHeapSize=29672, min timestamp=1733788623764, max timestamp=1733788623795Num uniques 1033; 2024-12-09T23:57:03,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47308, new segment data size=12958, suffix heap size=135400,new segment heap size=29672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1033, new segment cells count=283 2024-12-09T23:57:03,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47308, new segment data size=12958, suffix heap size=135400,new segment heap size=29672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1033, new segment cells count=283 2024-12-09T23:57:03,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=283, cellSize=12958, totalHeapSize=29672, min timestamp=1733788623764, max timestamp=1733788623795Num uniques 1033; 2024-12-09T23:57:03,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47308, new segment data size=12958, suffix heap size=135400,new segment heap size=29672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1033, new segment cells count=283 2024-12-09T23:57:03,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=242, cellSize=11082, totalHeapSize=25408, min timestamp=1733788623809, max timestamp=1733788623828Num uniques 1142; 2024-12-09T23:57:03,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52302, new segment data size=11082, suffix heap size=146736,new segment heap size=25408 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1142, new segment cells count=242 2024-12-09T23:57:03,832 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=242, cellSize=11082, totalHeapSize=25408, min timestamp=1733788623809, max timestamp=1733788623828Num uniques 1142; 2024-12-09T23:57:03,832 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52302, new segment data size=11082, suffix heap size=146736,new segment heap size=25408 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1142, new segment cells count=242 2024-12-09T23:57:03,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733788623809, max timestamp=1733788623828Num uniques 1142; 2024-12-09T23:57:03,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52302, new segment data size=6870, suffix heap size=146736,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1142, new segment cells count=150 2024-12-09T23:57:03,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=351, cellSize=16075, totalHeapSize=36744, min timestamp=1733788623851, max timestamp=1733788623865Num uniques 1009; 2024-12-09T23:57:03,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=16075, suffix heap size=132904,new segment heap size=36744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=351 2024-12-09T23:57:03,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=351, cellSize=16075, totalHeapSize=36744, min timestamp=1733788623851, max timestamp=1733788623865Num uniques 1101; 2024-12-09T23:57:03,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50425, new segment data size=16075, suffix heap size=142472,new segment heap size=36744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1101, new segment cells count=351 2024-12-09T23:57:03,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=351, cellSize=16075, totalHeapSize=36744, min timestamp=1733788623851, max timestamp=1733788623865Num uniques 1101; 2024-12-09T23:57:03,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50425, new segment data size=16075, suffix heap size=142472,new segment heap size=36744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1101, new segment cells count=351 2024-12-09T23:57:03,875 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=631 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/27d9acb9c4ad47cfaa8e271712dec620 2024-12-09T23:57:03,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/5a4ebebe99384c8d9a41459cc8b3cbc4 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/5a4ebebe99384c8d9a41459cc8b3cbc4 2024-12-09T23:57:03,890 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/5a4ebebe99384c8d9a41459cc8b3cbc4, entries=50, sequenceid=631, filesize=7.2 K 2024-12-09T23:57:03,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/af4166b40bd440dea035d4a356581f7d as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/af4166b40bd440dea035d4a356581f7d 2024-12-09T23:57:03,900 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/af4166b40bd440dea035d4a356581f7d, entries=50, sequenceid=631, filesize=7.2 K 2024-12-09T23:57:03,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/27d9acb9c4ad47cfaa8e271712dec620 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/27d9acb9c4ad47cfaa8e271712dec620 2024-12-09T23:57:03,908 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/27d9acb9c4ad47cfaa8e271712dec620, entries=50, sequenceid=631, filesize=7.2 K 2024-12-09T23:57:03,915 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=161.02 KB/164880 for c1cc0773bc61ee89c3c2791bfea1e203 in 1293ms, sequenceid=631, compaction requested=true 2024-12-09T23:57:03,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:57:03,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:57:03,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:03,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-09T23:57:03,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1cc0773bc61ee89c3c2791bfea1e203 3/3 column families, dataSize=167.72 KB heapSize=467.30 KB 2024-12-09T23:57:03,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-09T23:57:03,917 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=160, cellSize=7320, totalHeapSize=16880, min timestamp=1733788623898, max timestamp=1733788623906Num uniques 1210; 2024-12-09T23:57:03,917 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=160, cellSize=7320, totalHeapSize=16880, min timestamp=1733788623898, max timestamp=1733788623906Num uniques 1210; 2024-12-09T23:57:03,917 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=55410, new segment data size=7320, suffix heap size=153808,new segment heap size=16880 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1210, new segment cells count=160 2024-12-09T23:57:03,917 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=55410, new segment data size=7320, suffix heap size=153808,new segment heap size=16880 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1210, new segment cells count=160 2024-12-09T23:57:03,917 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=160, cellSize=7320, totalHeapSize=16880, min timestamp=1733788623898, max timestamp=1733788623906Num uniques 1210; 2024-12-09T23:57:03,917 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=55410, new segment data size=7320, suffix heap size=153808,new segment heap size=16880 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1210, new segment cells count=160 2024-12-09T23:57:03,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=A 2024-12-09T23:57:03,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:57:03,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=B 2024-12-09T23:57:03,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:57:03,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=C 2024-12-09T23:57:03,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:57:03,921 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-09T23:57:03,921 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0710 sec 2024-12-09T23:57:03,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/fa5d1c53689a413684ea6154ee6418a5 is 50, key is test_row_0/A:col10/1733788623904/Put/seqid=0 2024-12-09T23:57:03,931 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 2.0780 sec 2024-12-09T23:57:03,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741921_1097 (size=12301) 2024-12-09T23:57:03,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733788623927, max timestamp=1733788623945Num uniques 859; 2024-12-09T23:57:03,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-09T23:57:03,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733788623927, max timestamp=1733788623945Num uniques 859; 2024-12-09T23:57:03,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733788623927, max timestamp=1733788623945Num uniques 859; 2024-12-09T23:57:03,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-09T23:57:03,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-09T23:57:03,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:03,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 454 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788683965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:03,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:03,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:03,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:03,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:03,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 439 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788683971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:03,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:03,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:03,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 438 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788683973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:03,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:03,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:03,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:03,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:03,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:03,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T23:57:03,993 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:57:03,994 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:57:03,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-12-09T23:57:03,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T23:57:03,995 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:57:03,996 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:57:03,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:57:04,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:04,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 455 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788684083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:04,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:04,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 440 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788684083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:04,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:04,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:04,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:04,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:04,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 439 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788684083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:04,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:04,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:04,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:04,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:04,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:04,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:04,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T23:57:04,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T23:57:04,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:04,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:57:04,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:04,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:04,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:04,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:04,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:04,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:04,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 440 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788684293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:04,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 456 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788684293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:04,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:04,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:04,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 441 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788684293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:04,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:04,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:04,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:04,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:04,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:04,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:04,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:04,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:04,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T23:57:04,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:04,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:57:04,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:04,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:04,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:04,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:04,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T23:57:04,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=1323 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/fa5d1c53689a413684ea6154ee6418a5 2024-12-09T23:57:04,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/2565ddbcf50a4286a6fd4a0ec9de3d22 is 50, key is test_row_0/B:col10/1733788623904/Put/seqid=0 2024-12-09T23:57:04,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741922_1098 (size=12301) 2024-12-09T23:57:04,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T23:57:04,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:04,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:57:04,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:04,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:04,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:04,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:04,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:04,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49602 deadline: 1733788684562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:04,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:04,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1733788684562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:04,564 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_2' in c1cc0773bc61ee89c3c2791bfea1e203 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:57:04,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:04,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:04,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:04,564 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_1' in c1cc0773bc61ee89c3c2791bfea1e203 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:57:04,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:04,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:04,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:04,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:04,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 441 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788684603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:04,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 457 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788684603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:04,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 442 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788684603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:04,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T23:57:04,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:04,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:57:04,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:04,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:04,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:04,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:04,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T23:57:04,756 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=1323 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/2565ddbcf50a4286a6fd4a0ec9de3d22 2024-12-09T23:57:04,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T23:57:04,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:04,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:57:04,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:04,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:04,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:04,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/468d68496060493482091c8a2b55cc03 is 50, key is test_row_0/C:col10/1733788623904/Put/seqid=0 2024-12-09T23:57:04,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:04,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741923_1099 (size=12301) 2024-12-09T23:57:04,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T23:57:04,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:04,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:57:04,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:04,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:04,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:04,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:05,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T23:57:05,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:05,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:57:05,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:05,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:05,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:05,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:05,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:05,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 442 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788685113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:05,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:05,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 458 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788685113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:05,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:05,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:05,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:05,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:05,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:05,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:05,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:05,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 443 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788685113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:05,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:05,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:05,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:05,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T23:57:05,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=1323 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/468d68496060493482091c8a2b55cc03 2024-12-09T23:57:05,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/fa5d1c53689a413684ea6154ee6418a5 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/fa5d1c53689a413684ea6154ee6418a5 2024-12-09T23:57:05,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/fa5d1c53689a413684ea6154ee6418a5, entries=150, sequenceid=1323, filesize=12.0 K 2024-12-09T23:57:05,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/2565ddbcf50a4286a6fd4a0ec9de3d22 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/2565ddbcf50a4286a6fd4a0ec9de3d22 2024-12-09T23:57:05,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/2565ddbcf50a4286a6fd4a0ec9de3d22, entries=150, sequenceid=1323, filesize=12.0 K 2024-12-09T23:57:05,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/468d68496060493482091c8a2b55cc03 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/468d68496060493482091c8a2b55cc03 2024-12-09T23:57:05,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/468d68496060493482091c8a2b55cc03, entries=150, sequenceid=1323, filesize=12.0 K 2024-12-09T23:57:05,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.02 KB/88080, currentSize=161.02 KB/164880 for c1cc0773bc61ee89c3c2791bfea1e203 in 1282ms, sequenceid=1323, compaction requested=true 2024-12-09T23:57:05,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:57:05,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cc0773bc61ee89c3c2791bfea1e203:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:57:05,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:05,197 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T23:57:05,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cc0773bc61ee89c3c2791bfea1e203:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:57:05,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:05,198 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T23:57:05,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cc0773bc61ee89c3c2791bfea1e203:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:57:05,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:57:05,199 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45170 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T23:57:05,199 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45170 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T23:57:05,199 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): c1cc0773bc61ee89c3c2791bfea1e203/B is initiating minor compaction (all files) 2024-12-09T23:57:05,199 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): c1cc0773bc61ee89c3c2791bfea1e203/A is initiating minor compaction (all files) 2024-12-09T23:57:05,199 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cc0773bc61ee89c3c2791bfea1e203/B in TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:05,199 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cc0773bc61ee89c3c2791bfea1e203/A in TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:05,199 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/7267baa1e8d8443b97ad5d6f7dfcefb9, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/350c564b6de64730b6471bd90eea23cd, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/5a4ebebe99384c8d9a41459cc8b3cbc4, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/fa5d1c53689a413684ea6154ee6418a5] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp, totalSize=44.1 K 2024-12-09T23:57:05,199 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/ee2be71de7d548849bf265e97ff66b39, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/1b5ddeb6a364403daa70cad67cb9c7c1, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/af4166b40bd440dea035d4a356581f7d, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/2565ddbcf50a4286a6fd4a0ec9de3d22] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp, totalSize=44.1 K 2024-12-09T23:57:05,200 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7267baa1e8d8443b97ad5d6f7dfcefb9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=551, earliestPutTs=1733788621116 2024-12-09T23:57:05,200 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting ee2be71de7d548849bf265e97ff66b39, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=551, earliestPutTs=1733788621116 2024-12-09T23:57:05,200 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b5ddeb6a364403daa70cad67cb9c7c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=624, earliestPutTs=1733788621656 2024-12-09T23:57:05,200 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 350c564b6de64730b6471bd90eea23cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=624, earliestPutTs=1733788621656 2024-12-09T23:57:05,201 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting af4166b40bd440dea035d4a356581f7d, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=631, earliestPutTs=1733788621988 2024-12-09T23:57:05,201 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a4ebebe99384c8d9a41459cc8b3cbc4, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=631, earliestPutTs=1733788621988 2024-12-09T23:57:05,202 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 2565ddbcf50a4286a6fd4a0ec9de3d22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1323, earliestPutTs=1733788623898 2024-12-09T23:57:05,202 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa5d1c53689a413684ea6154ee6418a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1323, earliestPutTs=1733788623898 2024-12-09T23:57:05,217 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cc0773bc61ee89c3c2791bfea1e203#B#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:57:05,218 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/f328165578024d2d8617c3e57d8bb209 is 50, key is test_row_0/B:col10/1733788623904/Put/seqid=0 2024-12-09T23:57:05,219 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cc0773bc61ee89c3c2791bfea1e203#A#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:57:05,219 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/66174255ba8e42be85ecc63fc524e4bb is 50, key is test_row_0/A:col10/1733788623904/Put/seqid=0 2024-12-09T23:57:05,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T23:57:05,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:05,225 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing c1cc0773bc61ee89c3c2791bfea1e203 3/3 column families, dataSize=161.02 KB heapSize=441.23 KB 2024-12-09T23:57:05,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=A 2024-12-09T23:57:05,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:57:05,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=B 2024-12-09T23:57:05,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:57:05,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=C 2024-12-09T23:57:05,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:57:05,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741924_1100 (size=13289) 2024-12-09T23:57:05,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741925_1101 (size=13289) 2024-12-09T23:57:05,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/c31ea401e58e4623a5b944017868f5af is 50, key is test_row_0/A:col10/1733788623956/Put/seqid=0 2024-12-09T23:57:05,240 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/66174255ba8e42be85ecc63fc524e4bb as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/66174255ba8e42be85ecc63fc524e4bb 2024-12-09T23:57:05,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741926_1102 (size=12301) 2024-12-09T23:57:05,248 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1cc0773bc61ee89c3c2791bfea1e203/A of c1cc0773bc61ee89c3c2791bfea1e203 into 66174255ba8e42be85ecc63fc524e4bb(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:05,248 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:57:05,248 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., storeName=c1cc0773bc61ee89c3c2791bfea1e203/A, priority=12, startTime=1733788625197; duration=0sec 2024-12-09T23:57:05,248 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:57:05,248 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cc0773bc61ee89c3c2791bfea1e203:A 2024-12-09T23:57:05,248 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T23:57:05,250 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45170 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T23:57:05,250 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): c1cc0773bc61ee89c3c2791bfea1e203/C is initiating minor compaction (all files) 2024-12-09T23:57:05,250 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cc0773bc61ee89c3c2791bfea1e203/C in TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:05,250 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/670be54c72754068aba66da9f58dafb9, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/2ad8aac4fc2d4f3d9070e9eabdab07be, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/27d9acb9c4ad47cfaa8e271712dec620, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/468d68496060493482091c8a2b55cc03] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp, totalSize=44.1 K 2024-12-09T23:57:05,250 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 670be54c72754068aba66da9f58dafb9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=551, earliestPutTs=1733788621116 2024-12-09T23:57:05,251 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ad8aac4fc2d4f3d9070e9eabdab07be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=624, earliestPutTs=1733788621656 2024-12-09T23:57:05,252 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27d9acb9c4ad47cfaa8e271712dec620, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=631, earliestPutTs=1733788621988 2024-12-09T23:57:05,253 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 468d68496060493482091c8a2b55cc03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1323, earliestPutTs=1733788623898 2024-12-09T23:57:05,265 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cc0773bc61ee89c3c2791bfea1e203#C#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:57:05,266 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/97bf413680a94a15b22dcf3c0c4cc391 is 50, key is test_row_0/C:col10/1733788623904/Put/seqid=0 2024-12-09T23:57:05,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741927_1103 (size=13289) 2024-12-09T23:57:05,636 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/f328165578024d2d8617c3e57d8bb209 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/f328165578024d2d8617c3e57d8bb209 2024-12-09T23:57:05,641 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1cc0773bc61ee89c3c2791bfea1e203/B of c1cc0773bc61ee89c3c2791bfea1e203 into f328165578024d2d8617c3e57d8bb209(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:05,641 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:57:05,641 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., storeName=c1cc0773bc61ee89c3c2791bfea1e203/B, priority=12, startTime=1733788625197; duration=0sec 2024-12-09T23:57:05,641 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:05,641 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cc0773bc61ee89c3c2791bfea1e203:B 2024-12-09T23:57:05,644 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=1359 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/c31ea401e58e4623a5b944017868f5af 2024-12-09T23:57:05,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/81d5bc625fcd49bebe0e661df17e5a8f is 50, key is test_row_0/B:col10/1733788623956/Put/seqid=0 2024-12-09T23:57:05,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741928_1104 (size=12301) 2024-12-09T23:57:05,678 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/97bf413680a94a15b22dcf3c0c4cc391 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/97bf413680a94a15b22dcf3c0c4cc391 2024-12-09T23:57:05,684 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1cc0773bc61ee89c3c2791bfea1e203/C of c1cc0773bc61ee89c3c2791bfea1e203 into 97bf413680a94a15b22dcf3c0c4cc391(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:05,684 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:57:05,684 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., storeName=c1cc0773bc61ee89c3c2791bfea1e203/C, priority=12, startTime=1733788625198; duration=0sec 2024-12-09T23:57:05,684 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:05,684 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cc0773bc61ee89c3c2791bfea1e203:C 2024-12-09T23:57:06,058 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=1359 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/81d5bc625fcd49bebe0e661df17e5a8f 2024-12-09T23:57:06,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/d54d9e07346f48b690b8b7a597030a1f is 50, key is test_row_0/C:col10/1733788623956/Put/seqid=0 2024-12-09T23:57:06,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741929_1105 (size=12301) 2024-12-09T23:57:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:57:06,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. as already flushing 2024-12-09T23:57:06,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 461 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788686127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:06,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:06,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:06,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:06,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 446 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788686129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:06,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:06,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:06,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:06,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:06,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:06,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 443 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49646 deadline: 1733788686133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:06,134 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_2' in c1cc0773bc61ee89c3c2791bfea1e203 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:57:06,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:06,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:06,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T23:57:06,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 462 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788686233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:06,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:06,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:06,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:06,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 447 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788686233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:06,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:06,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:06,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:06,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:06,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 463 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49604 deadline: 1733788686442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:06,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:06,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:06,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:06,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:06,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 448 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49630 deadline: 1733788686442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:06,446 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:06,446 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:06,446 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:06,472 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=1359 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/d54d9e07346f48b690b8b7a597030a1f 2024-12-09T23:57:06,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/c31ea401e58e4623a5b944017868f5af as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/c31ea401e58e4623a5b944017868f5af 2024-12-09T23:57:06,486 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/c31ea401e58e4623a5b944017868f5af, entries=150, sequenceid=1359, filesize=12.0 K 2024-12-09T23:57:06,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/81d5bc625fcd49bebe0e661df17e5a8f as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/81d5bc625fcd49bebe0e661df17e5a8f 2024-12-09T23:57:06,493 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/81d5bc625fcd49bebe0e661df17e5a8f, entries=150, sequenceid=1359, filesize=12.0 K 2024-12-09T23:57:06,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/d54d9e07346f48b690b8b7a597030a1f as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/d54d9e07346f48b690b8b7a597030a1f 2024-12-09T23:57:06,500 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/d54d9e07346f48b690b8b7a597030a1f, entries=150, sequenceid=1359, filesize=12.0 K 2024-12-09T23:57:06,501 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~441.19 KB/451776, currentSize=26.84 KB/27480 for c1cc0773bc61ee89c3c2791bfea1e203 in 1276ms, sequenceid=1359, compaction requested=false 2024-12-09T23:57:06,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:57:06,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:06,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-12-09T23:57:06,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-12-09T23:57:06,507 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-09T23:57:06,507 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5070 sec 2024-12-09T23:57:06,509 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.5130 sec 2024-12-09T23:57:06,727 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-09T23:57:06,727 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-09T23:57:06,728 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:57:06,728 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:57:06,728 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-09T23:57:06,728 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:57:06,730 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-09T23:57:06,730 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:57:06,731 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:06,731 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:06,731 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:06,731 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:06,732 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:06,732 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:06,732 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:06,732 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:06,732 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:57:06,732 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:57:06,732 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:57:06,732 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:57:06,755 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-09T23:57:06,755 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-09T23:57:06,755 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:57:06,755 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:57:06,755 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:06,755 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:06,755 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:06,755 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:06,755 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:57:06,755 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:57:08,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:57:08,158 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1cc0773bc61ee89c3c2791bfea1e203 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T23:57:08,159 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-09T23:57:08,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=A 2024-12-09T23:57:08,159 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:57:08,159 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:08,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:08,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=B 2024-12-09T23:57:08,160 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:08,160 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:57:08,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:08,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=C 2024-12-09T23:57:08,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:08,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T23:57:08,163 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:57:08,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/4a84e8881b8e46d8aa53f196092e5037 is 50, key is test_row_1/A:col10/1733788626124/Put/seqid=0 2024-12-09T23:57:08,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741930_1106 (size=9857) 2024-12-09T23:57:08,291 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T23:57:08,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=1372 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/4a84e8881b8e46d8aa53f196092e5037 2024-12-09T23:57:08,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/689ac4a03e554a4bb650bfaaae557997 is 50, key is test_row_1/B:col10/1733788626124/Put/seqid=0 2024-12-09T23:57:08,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741931_1107 (size=9857) 2024-12-09T23:57:08,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=1372 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/689ac4a03e554a4bb650bfaaae557997 2024-12-09T23:57:09,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/2fabd41320604163b5d68adc892b539c is 50, key is test_row_1/C:col10/1733788626124/Put/seqid=0 2024-12-09T23:57:09,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741932_1108 (size=9857) 2024-12-09T23:57:09,414 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=1372 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/2fabd41320604163b5d68adc892b539c 2024-12-09T23:57:09,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/4a84e8881b8e46d8aa53f196092e5037 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/4a84e8881b8e46d8aa53f196092e5037 2024-12-09T23:57:09,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/4a84e8881b8e46d8aa53f196092e5037, entries=100, sequenceid=1372, filesize=9.6 K 2024-12-09T23:57:09,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/689ac4a03e554a4bb650bfaaae557997 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/689ac4a03e554a4bb650bfaaae557997 2024-12-09T23:57:09,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/689ac4a03e554a4bb650bfaaae557997, entries=100, sequenceid=1372, filesize=9.6 K 2024-12-09T23:57:09,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/2fabd41320604163b5d68adc892b539c as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/2fabd41320604163b5d68adc892b539c 2024-12-09T23:57:09,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/2fabd41320604163b5d68adc892b539c, entries=100, sequenceid=1372, filesize=9.6 K 2024-12-09T23:57:09,442 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for c1cc0773bc61ee89c3c2791bfea1e203 in 1284ms, sequenceid=1372, compaction requested=true 2024-12-09T23:57:09,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:57:09,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cc0773bc61ee89c3c2791bfea1e203:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:57:09,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:09,442 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:57:09,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cc0773bc61ee89c3c2791bfea1e203:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:57:09,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:09,443 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:57:09,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1cc0773bc61ee89c3c2791bfea1e203:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:57:09,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:57:09,444 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:57:09,444 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:57:09,444 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): c1cc0773bc61ee89c3c2791bfea1e203/B is initiating minor compaction (all files) 2024-12-09T23:57:09,444 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): c1cc0773bc61ee89c3c2791bfea1e203/A is initiating minor compaction (all files) 2024-12-09T23:57:09,444 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cc0773bc61ee89c3c2791bfea1e203/B in TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:09,444 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cc0773bc61ee89c3c2791bfea1e203/A in TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:09,444 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/f328165578024d2d8617c3e57d8bb209, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/81d5bc625fcd49bebe0e661df17e5a8f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/689ac4a03e554a4bb650bfaaae557997] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp, totalSize=34.6 K 2024-12-09T23:57:09,444 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/66174255ba8e42be85ecc63fc524e4bb, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/c31ea401e58e4623a5b944017868f5af, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/4a84e8881b8e46d8aa53f196092e5037] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp, totalSize=34.6 K 2024-12-09T23:57:09,444 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66174255ba8e42be85ecc63fc524e4bb, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=1323, earliestPutTs=1733788623898 2024-12-09T23:57:09,444 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting f328165578024d2d8617c3e57d8bb209, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=1323, earliestPutTs=1733788623898 2024-12-09T23:57:09,444 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 81d5bc625fcd49bebe0e661df17e5a8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1359, earliestPutTs=1733788623956 2024-12-09T23:57:09,444 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting c31ea401e58e4623a5b944017868f5af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1359, earliestPutTs=1733788623956 2024-12-09T23:57:09,445 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 689ac4a03e554a4bb650bfaaae557997, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=1372, earliestPutTs=1733788626124 2024-12-09T23:57:09,445 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a84e8881b8e46d8aa53f196092e5037, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=1372, earliestPutTs=1733788626124 2024-12-09T23:57:09,451 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cc0773bc61ee89c3c2791bfea1e203#A#compaction#97 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:57:09,451 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cc0773bc61ee89c3c2791bfea1e203#B#compaction#96 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:57:09,452 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/213dcfc5170c4853b586d8105400b695 is 50, key is test_row_0/A:col10/1733788623956/Put/seqid=0 2024-12-09T23:57:09,452 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/e5c9ffbe1b6a4bff9800fcaac2c7daad is 50, key is test_row_0/B:col10/1733788623956/Put/seqid=0 2024-12-09T23:57:09,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741933_1109 (size=13391) 2024-12-09T23:57:09,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741934_1110 (size=13391) 2024-12-09T23:57:09,861 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/213dcfc5170c4853b586d8105400b695 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/213dcfc5170c4853b586d8105400b695 2024-12-09T23:57:09,861 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/e5c9ffbe1b6a4bff9800fcaac2c7daad as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/e5c9ffbe1b6a4bff9800fcaac2c7daad 2024-12-09T23:57:09,867 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1cc0773bc61ee89c3c2791bfea1e203/A of c1cc0773bc61ee89c3c2791bfea1e203 into 213dcfc5170c4853b586d8105400b695(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:09,867 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1cc0773bc61ee89c3c2791bfea1e203/B of c1cc0773bc61ee89c3c2791bfea1e203 into e5c9ffbe1b6a4bff9800fcaac2c7daad(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:09,867 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:57:09,867 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:57:09,867 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., storeName=c1cc0773bc61ee89c3c2791bfea1e203/A, priority=13, startTime=1733788629442; duration=0sec 2024-12-09T23:57:09,867 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., storeName=c1cc0773bc61ee89c3c2791bfea1e203/B, priority=13, startTime=1733788629442; duration=0sec 2024-12-09T23:57:09,867 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:57:09,867 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cc0773bc61ee89c3c2791bfea1e203:A 2024-12-09T23:57:09,867 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:57:09,867 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cc0773bc61ee89c3c2791bfea1e203:B 2024-12-09T23:57:09,867 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:57:09,868 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:57:09,868 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): c1cc0773bc61ee89c3c2791bfea1e203/C is initiating minor compaction (all files) 2024-12-09T23:57:09,868 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1cc0773bc61ee89c3c2791bfea1e203/C in TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:09,868 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/97bf413680a94a15b22dcf3c0c4cc391, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/d54d9e07346f48b690b8b7a597030a1f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/2fabd41320604163b5d68adc892b539c] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp, totalSize=34.6 K 2024-12-09T23:57:09,869 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97bf413680a94a15b22dcf3c0c4cc391, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=1323, earliestPutTs=1733788623898 2024-12-09T23:57:09,869 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting d54d9e07346f48b690b8b7a597030a1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1359, earliestPutTs=1733788623956 2024-12-09T23:57:09,869 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fabd41320604163b5d68adc892b539c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=1372, earliestPutTs=1733788626124 2024-12-09T23:57:09,877 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1cc0773bc61ee89c3c2791bfea1e203#C#compaction#98 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:57:09,877 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/499b65ea6f8c481286f137266a4b7f24 is 50, key is test_row_0/C:col10/1733788623956/Put/seqid=0 2024-12-09T23:57:09,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741935_1111 (size=13391) 2024-12-09T23:57:10,294 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/499b65ea6f8c481286f137266a4b7f24 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/499b65ea6f8c481286f137266a4b7f24 2024-12-09T23:57:10,300 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1cc0773bc61ee89c3c2791bfea1e203/C of c1cc0773bc61ee89c3c2791bfea1e203 into 499b65ea6f8c481286f137266a4b7f24(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:10,300 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1cc0773bc61ee89c3c2791bfea1e203: 2024-12-09T23:57:10,301 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., storeName=c1cc0773bc61ee89c3c2791bfea1e203/C, priority=13, startTime=1733788629443; duration=0sec 2024-12-09T23:57:10,301 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:10,301 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1cc0773bc61ee89c3c2791bfea1e203:C 2024-12-09T23:57:14,608 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-09T23:57:14,608 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:57:14,608 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:14,609 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:14,609 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:57:14,624 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-09T23:57:14,624 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:57:14,624 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:14,624 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:14,624 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:57:14,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T23:57:14,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-12-09T23:57:14,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 382 2024-12-09T23:57:14,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-12-09T23:57:14,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 397 2024-12-09T23:57:14,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 379 2024-12-09T23:57:14,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T23:57:14,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4035 2024-12-09T23:57:14,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3990 2024-12-09T23:57:14,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T23:57:14,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1771 2024-12-09T23:57:14,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5308 rows 2024-12-09T23:57:14,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1781 2024-12-09T23:57:14,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5342 rows 2024-12-09T23:57:14,625 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T23:57:14,625 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T23:57:14,625 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:14,625 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:14,625 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:57:14,629 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-09T23:57:14,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T23:57:14,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T23:57:14,635 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733788634635"}]},"ts":"1733788634635"} 2024-12-09T23:57:14,637 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T23:57:14,637 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T23:57:14,638 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T23:57:14,642 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1cc0773bc61ee89c3c2791bfea1e203, UNASSIGN}] 2024-12-09T23:57:14,643 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1cc0773bc61ee89c3c2791bfea1e203, UNASSIGN 2024-12-09T23:57:14,645 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=c1cc0773bc61ee89c3c2791bfea1e203, regionState=CLOSING, regionLocation=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:14,646 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1cc0773bc61ee89c3c2791bfea1e203, UNASSIGN because future has completed 2024-12-09T23:57:14,647 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T23:57:14,647 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837}] 2024-12-09T23:57:14,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T23:57:14,807 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:57:14,807 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T23:57:14,808 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing c1cc0773bc61ee89c3c2791bfea1e203, disabling compactions & flushes 2024-12-09T23:57:14,809 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:14,809 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:14,809 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. after waiting 0 ms 2024-12-09T23:57:14,809 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:14,809 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing c1cc0773bc61ee89c3c2791bfea1e203 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-12-09T23:57:14,810 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=A 2024-12-09T23:57:14,810 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:14,810 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=B 2024-12-09T23:57:14,810 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:14,810 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1cc0773bc61ee89c3c2791bfea1e203, store=C 2024-12-09T23:57:14,810 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:14,816 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/af29573d693f4f9397632b9835f4e86e is 50, key is test_row_1/A:col10/1733788634623/Put/seqid=0 2024-12-09T23:57:14,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741936_1112 (size=9857) 2024-12-09T23:57:14,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T23:57:15,223 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=1380 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/af29573d693f4f9397632b9835f4e86e 2024-12-09T23:57:15,235 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/5f2ae540c5484c9495db6d8435cc3379 is 50, key is test_row_1/B:col10/1733788634623/Put/seqid=0 2024-12-09T23:57:15,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741937_1113 (size=9857) 2024-12-09T23:57:15,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T23:57:15,642 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=1380 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/5f2ae540c5484c9495db6d8435cc3379 2024-12-09T23:57:15,654 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/81ec5c6145a343f69e0c217605be9ed9 is 50, key is test_row_1/C:col10/1733788634623/Put/seqid=0 2024-12-09T23:57:15,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741938_1114 (size=9857) 2024-12-09T23:57:15,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T23:57:16,061 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=1380 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/81ec5c6145a343f69e0c217605be9ed9 2024-12-09T23:57:16,070 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/A/af29573d693f4f9397632b9835f4e86e as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/af29573d693f4f9397632b9835f4e86e 2024-12-09T23:57:16,075 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/af29573d693f4f9397632b9835f4e86e, entries=100, sequenceid=1380, filesize=9.6 K 2024-12-09T23:57:16,076 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/B/5f2ae540c5484c9495db6d8435cc3379 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/5f2ae540c5484c9495db6d8435cc3379 2024-12-09T23:57:16,081 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/5f2ae540c5484c9495db6d8435cc3379, entries=100, sequenceid=1380, filesize=9.6 K 2024-12-09T23:57:16,083 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/.tmp/C/81ec5c6145a343f69e0c217605be9ed9 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/81ec5c6145a343f69e0c217605be9ed9 2024-12-09T23:57:16,088 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/81ec5c6145a343f69e0c217605be9ed9, entries=100, sequenceid=1380, filesize=9.6 K 2024-12-09T23:57:16,089 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for c1cc0773bc61ee89c3c2791bfea1e203 in 1279ms, sequenceid=1380, compaction requested=false 2024-12-09T23:57:16,089 DEBUG [StoreCloser-TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/635f28a26aea4f9bbb9cf7081136bb66, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/3b31be394fd84f488e8f169c06213427, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/e201be93a15a4e849d4f510a182bf4f4, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/d8be3c94d55b4eb780ab9df802fb9118, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/7c011deb065146f0b1f95ea8abb31f92, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/072c8074462a4f57bb86f403a64151f9, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/4effd93271284b24a615b19edd357a87, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/47d5324a9de64e10a2b1ebb5da6ef060, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/e4f7a6da235445639ceb3567c3b8dfb7, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/37285f2fc5cb4fc7a9537f1dcb4e8de2, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/b63153ee1caf46848b214f3abecdead8, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/e0b1e0d2822e4b79a253df2598869092, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/5a25a8684b2947d281267fe6033be094, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/863c4f5fd3ea4dcfa87cfeec2f36c032, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/6b1fcd8d61494a129911ee981010afd7, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/3fb385eddd164eec9af01ec68c430fa4, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/c97d3c65c60e49069215a23776b2ab84, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/62b20da770ee4afeaf7d23238e9657f1, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/4d2af723ad5d4c348340f68f6baac4d2, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/2199f0d158e34811b1aa396038ad5642, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/cb9cfc69e047401f919135ff4226fa42, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/5ed8c19b712748a5bb86b9e109eb1448, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/c77cefcbc7bb4c27856960b8890c9896, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/6684d32bb202407894924b42f51b0681, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/e89da75ab5d44e06bd24a1d45d076457, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/7267baa1e8d8443b97ad5d6f7dfcefb9, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/350c564b6de64730b6471bd90eea23cd, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/5a4ebebe99384c8d9a41459cc8b3cbc4, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/66174255ba8e42be85ecc63fc524e4bb, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/fa5d1c53689a413684ea6154ee6418a5, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/c31ea401e58e4623a5b944017868f5af, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/4a84e8881b8e46d8aa53f196092e5037] to archive 2024-12-09T23:57:16,093 DEBUG [StoreCloser-TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-09T23:57:16,102 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/635f28a26aea4f9bbb9cf7081136bb66 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/635f28a26aea4f9bbb9cf7081136bb66 2024-12-09T23:57:16,102 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/7c011deb065146f0b1f95ea8abb31f92 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/7c011deb065146f0b1f95ea8abb31f92 2024-12-09T23:57:16,102 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/e201be93a15a4e849d4f510a182bf4f4 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/e201be93a15a4e849d4f510a182bf4f4 2024-12-09T23:57:16,102 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/47d5324a9de64e10a2b1ebb5da6ef060 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/47d5324a9de64e10a2b1ebb5da6ef060 2024-12-09T23:57:16,103 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/4effd93271284b24a615b19edd357a87 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/4effd93271284b24a615b19edd357a87 2024-12-09T23:57:16,103 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/3b31be394fd84f488e8f169c06213427 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/3b31be394fd84f488e8f169c06213427 2024-12-09T23:57:16,103 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/d8be3c94d55b4eb780ab9df802fb9118 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/d8be3c94d55b4eb780ab9df802fb9118 2024-12-09T23:57:16,103 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/072c8074462a4f57bb86f403a64151f9 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/072c8074462a4f57bb86f403a64151f9 2024-12-09T23:57:16,105 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/6b1fcd8d61494a129911ee981010afd7 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/6b1fcd8d61494a129911ee981010afd7 2024-12-09T23:57:16,105 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/5a25a8684b2947d281267fe6033be094 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/5a25a8684b2947d281267fe6033be094 2024-12-09T23:57:16,105 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/e4f7a6da235445639ceb3567c3b8dfb7 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/e4f7a6da235445639ceb3567c3b8dfb7 2024-12-09T23:57:16,105 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/b63153ee1caf46848b214f3abecdead8 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/b63153ee1caf46848b214f3abecdead8 2024-12-09T23:57:16,105 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/37285f2fc5cb4fc7a9537f1dcb4e8de2 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/37285f2fc5cb4fc7a9537f1dcb4e8de2 2024-12-09T23:57:16,107 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/3fb385eddd164eec9af01ec68c430fa4 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/3fb385eddd164eec9af01ec68c430fa4 2024-12-09T23:57:16,108 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/863c4f5fd3ea4dcfa87cfeec2f36c032 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/863c4f5fd3ea4dcfa87cfeec2f36c032 2024-12-09T23:57:16,109 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/c97d3c65c60e49069215a23776b2ab84 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/c97d3c65c60e49069215a23776b2ab84 2024-12-09T23:57:16,109 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/e0b1e0d2822e4b79a253df2598869092 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/e0b1e0d2822e4b79a253df2598869092 2024-12-09T23:57:16,109 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/2199f0d158e34811b1aa396038ad5642 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/2199f0d158e34811b1aa396038ad5642 2024-12-09T23:57:16,109 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/62b20da770ee4afeaf7d23238e9657f1 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/62b20da770ee4afeaf7d23238e9657f1 2024-12-09T23:57:16,110 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/4d2af723ad5d4c348340f68f6baac4d2 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/4d2af723ad5d4c348340f68f6baac4d2 2024-12-09T23:57:16,110 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/c77cefcbc7bb4c27856960b8890c9896 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/c77cefcbc7bb4c27856960b8890c9896 2024-12-09T23:57:16,110 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/cb9cfc69e047401f919135ff4226fa42 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/cb9cfc69e047401f919135ff4226fa42 2024-12-09T23:57:16,111 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/5ed8c19b712748a5bb86b9e109eb1448 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/5ed8c19b712748a5bb86b9e109eb1448 2024-12-09T23:57:16,111 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/7267baa1e8d8443b97ad5d6f7dfcefb9 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/7267baa1e8d8443b97ad5d6f7dfcefb9 2024-12-09T23:57:16,112 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/6684d32bb202407894924b42f51b0681 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/6684d32bb202407894924b42f51b0681 2024-12-09T23:57:16,112 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/350c564b6de64730b6471bd90eea23cd to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/350c564b6de64730b6471bd90eea23cd 2024-12-09T23:57:16,112 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/5a4ebebe99384c8d9a41459cc8b3cbc4 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/5a4ebebe99384c8d9a41459cc8b3cbc4 2024-12-09T23:57:16,112 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/66174255ba8e42be85ecc63fc524e4bb to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/66174255ba8e42be85ecc63fc524e4bb 2024-12-09T23:57:16,113 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/4a84e8881b8e46d8aa53f196092e5037 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/4a84e8881b8e46d8aa53f196092e5037 2024-12-09T23:57:16,113 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/e89da75ab5d44e06bd24a1d45d076457 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/e89da75ab5d44e06bd24a1d45d076457 2024-12-09T23:57:16,113 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/c31ea401e58e4623a5b944017868f5af to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/c31ea401e58e4623a5b944017868f5af 2024-12-09T23:57:16,113 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/fa5d1c53689a413684ea6154ee6418a5 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/fa5d1c53689a413684ea6154ee6418a5 2024-12-09T23:57:16,124 DEBUG [StoreCloser-TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/80de9e803b58411c9848a24e09f80ade, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/01b88dc5dc924475ab73054189622e83, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/6e72ff982971413bbc64347fe85f54e1, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/16683551c7254d00b99ad62d675b606b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/ea4b6c50d35844cd92a95822eb5e776b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/86bb984ba4234388b1c3053deddcf95b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/dbe2e15349d9454b8f6e43c3cd51cc94, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/c495ca8982e24f77a907035b3638ad36, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/b9ebb4e000bf42c196be0a5cbb407c4b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/829bf64bb7c6473c919d0ee6a724c34f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/604714add4c04d6b9e9f92974a1b2f50, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/8cdb31daff61487c8f83dd093c649cdd, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/4bf9353b8da24d87a28682059c103f6d, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/b77050f8cd414467a530b6609055600c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/c5278d28aed74da7a8cbca3a647f06c1, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/3dcdea4cf6ba4696978a837e15150266, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/813de5dff76243bda587f1507094e889, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/3ebd779e73e5460d8c4c588d44c5a746, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/eaa13bb5911a4c3a8a658fa6adffae32, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/ec8e8e72066a44c499254bd9fdd54b04, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/fbd409cff57042789d6d022f1f285fed, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/10e637f4b15a4d7eb5bb22fb23d113a8, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/c1a717c08f8b43938ed59fc67a32e0b4, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/94baea67ce184506b5fd6c109b0aba4c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/ee2be71de7d548849bf265e97ff66b39, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/d345503003744e3b9591b9b43ba15502, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/1b5ddeb6a364403daa70cad67cb9c7c1, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/af4166b40bd440dea035d4a356581f7d, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/f328165578024d2d8617c3e57d8bb209, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/2565ddbcf50a4286a6fd4a0ec9de3d22, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/81d5bc625fcd49bebe0e661df17e5a8f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/689ac4a03e554a4bb650bfaaae557997] to archive 2024-12-09T23:57:16,125 DEBUG [StoreCloser-TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-09T23:57:16,127 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/16683551c7254d00b99ad62d675b606b to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/16683551c7254d00b99ad62d675b606b 2024-12-09T23:57:16,127 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/80de9e803b58411c9848a24e09f80ade to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/80de9e803b58411c9848a24e09f80ade 2024-12-09T23:57:16,127 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/6e72ff982971413bbc64347fe85f54e1 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/6e72ff982971413bbc64347fe85f54e1 2024-12-09T23:57:16,128 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/ea4b6c50d35844cd92a95822eb5e776b to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/ea4b6c50d35844cd92a95822eb5e776b 2024-12-09T23:57:16,128 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/c495ca8982e24f77a907035b3638ad36 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/c495ca8982e24f77a907035b3638ad36 2024-12-09T23:57:16,128 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/01b88dc5dc924475ab73054189622e83 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/01b88dc5dc924475ab73054189622e83 2024-12-09T23:57:16,128 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/dbe2e15349d9454b8f6e43c3cd51cc94 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/dbe2e15349d9454b8f6e43c3cd51cc94 2024-12-09T23:57:16,129 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/86bb984ba4234388b1c3053deddcf95b to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/86bb984ba4234388b1c3053deddcf95b 2024-12-09T23:57:16,129 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/829bf64bb7c6473c919d0ee6a724c34f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/829bf64bb7c6473c919d0ee6a724c34f 2024-12-09T23:57:16,129 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/b9ebb4e000bf42c196be0a5cbb407c4b to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/b9ebb4e000bf42c196be0a5cbb407c4b 2024-12-09T23:57:16,130 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/8cdb31daff61487c8f83dd093c649cdd to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/8cdb31daff61487c8f83dd093c649cdd 2024-12-09T23:57:16,130 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/4bf9353b8da24d87a28682059c103f6d to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/4bf9353b8da24d87a28682059c103f6d 2024-12-09T23:57:16,130 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/604714add4c04d6b9e9f92974a1b2f50 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/604714add4c04d6b9e9f92974a1b2f50 2024-12-09T23:57:16,130 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/b77050f8cd414467a530b6609055600c to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/b77050f8cd414467a530b6609055600c 2024-12-09T23:57:16,130 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/c5278d28aed74da7a8cbca3a647f06c1 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/c5278d28aed74da7a8cbca3a647f06c1 2024-12-09T23:57:16,131 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/3dcdea4cf6ba4696978a837e15150266 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/3dcdea4cf6ba4696978a837e15150266 2024-12-09T23:57:16,131 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/3ebd779e73e5460d8c4c588d44c5a746 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/3ebd779e73e5460d8c4c588d44c5a746 2024-12-09T23:57:16,132 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/eaa13bb5911a4c3a8a658fa6adffae32 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/eaa13bb5911a4c3a8a658fa6adffae32 2024-12-09T23:57:16,132 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/ec8e8e72066a44c499254bd9fdd54b04 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/ec8e8e72066a44c499254bd9fdd54b04 2024-12-09T23:57:16,132 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/c1a717c08f8b43938ed59fc67a32e0b4 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/c1a717c08f8b43938ed59fc67a32e0b4 2024-12-09T23:57:16,132 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/fbd409cff57042789d6d022f1f285fed to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/fbd409cff57042789d6d022f1f285fed 2024-12-09T23:57:16,132 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/10e637f4b15a4d7eb5bb22fb23d113a8 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/10e637f4b15a4d7eb5bb22fb23d113a8 2024-12-09T23:57:16,133 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/ee2be71de7d548849bf265e97ff66b39 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/ee2be71de7d548849bf265e97ff66b39 2024-12-09T23:57:16,133 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/94baea67ce184506b5fd6c109b0aba4c to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/94baea67ce184506b5fd6c109b0aba4c 2024-12-09T23:57:16,134 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/f328165578024d2d8617c3e57d8bb209 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/f328165578024d2d8617c3e57d8bb209 2024-12-09T23:57:16,134 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/af4166b40bd440dea035d4a356581f7d to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/af4166b40bd440dea035d4a356581f7d 2024-12-09T23:57:16,134 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/1b5ddeb6a364403daa70cad67cb9c7c1 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/1b5ddeb6a364403daa70cad67cb9c7c1 2024-12-09T23:57:16,134 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/d345503003744e3b9591b9b43ba15502 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/d345503003744e3b9591b9b43ba15502 2024-12-09T23:57:16,134 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/2565ddbcf50a4286a6fd4a0ec9de3d22 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/2565ddbcf50a4286a6fd4a0ec9de3d22 2024-12-09T23:57:16,135 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/81d5bc625fcd49bebe0e661df17e5a8f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/81d5bc625fcd49bebe0e661df17e5a8f 2024-12-09T23:57:16,135 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/689ac4a03e554a4bb650bfaaae557997 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/689ac4a03e554a4bb650bfaaae557997 2024-12-09T23:57:16,137 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/813de5dff76243bda587f1507094e889 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/813de5dff76243bda587f1507094e889 2024-12-09T23:57:16,139 DEBUG [StoreCloser-TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/7d27212aede44f18adb8034aef636699, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/34317ca0c5624706958db763e910f8a3, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/9c07739a9df14ea48f85c5034a9aac56, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/ae85d0fd792f46d88761cb60dedb7791, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/9df03c58c0234258a376de4fcb864992, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/b0d319bcd4ab4affb6abe8c048d6a5fe, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/9c88d343dd2e48289012fb6b29275f82, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/02106665c08846b2a70c99ec3933ddc9, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/82d9ff3835774c28b2895003afc77621, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/1e6c5a1161d140b59d3ab1d8d115553b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/2f5c5d6020824b498f4df3e2c18ac607, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/6765a71eb8794558b2a177f60c1e88bf, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/37b87d3be795411f9ca2b277f5c526c7, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/422ff75ace6f4858bd7ccd1f135d4405, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/0cdb3add422045c8bd9b8197450f2ed8, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/00743dc7048145f886e636466cde182f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/65badc2a4c4441bab56ce32b344a214b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/150e3281f14a481fb8885e7224e2bd4f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/f8f11dbb1011486aa0c068cf2ae6399b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/1d7ac4291d624ce281a45aa2a1a34e76, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/d67e2984e27b4b878ec4a64a77dd1ad3, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/9aeabdedfd744be69105f074affa760c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/b2d7abf442924cad8e32688a1dee6401, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/95d8fa7d58fc442387783f6d36215758, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/670be54c72754068aba66da9f58dafb9, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/3c822ca84e4d4272b3bbbf0e13be22e2, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/2ad8aac4fc2d4f3d9070e9eabdab07be, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/27d9acb9c4ad47cfaa8e271712dec620, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/97bf413680a94a15b22dcf3c0c4cc391, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/468d68496060493482091c8a2b55cc03, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/d54d9e07346f48b690b8b7a597030a1f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/2fabd41320604163b5d68adc892b539c] to archive 2024-12-09T23:57:16,140 DEBUG [StoreCloser-TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-09T23:57:16,142 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/7d27212aede44f18adb8034aef636699 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/7d27212aede44f18adb8034aef636699 2024-12-09T23:57:16,142 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/9df03c58c0234258a376de4fcb864992 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/9df03c58c0234258a376de4fcb864992 2024-12-09T23:57:16,142 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/ae85d0fd792f46d88761cb60dedb7791 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/ae85d0fd792f46d88761cb60dedb7791 2024-12-09T23:57:16,142 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/02106665c08846b2a70c99ec3933ddc9 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/02106665c08846b2a70c99ec3933ddc9 2024-12-09T23:57:16,142 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/9c88d343dd2e48289012fb6b29275f82 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/9c88d343dd2e48289012fb6b29275f82 2024-12-09T23:57:16,142 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/b0d319bcd4ab4affb6abe8c048d6a5fe to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/b0d319bcd4ab4affb6abe8c048d6a5fe 2024-12-09T23:57:16,144 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/2f5c5d6020824b498f4df3e2c18ac607 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/2f5c5d6020824b498f4df3e2c18ac607 2024-12-09T23:57:16,144 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/1e6c5a1161d140b59d3ab1d8d115553b to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/1e6c5a1161d140b59d3ab1d8d115553b 2024-12-09T23:57:16,144 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/6765a71eb8794558b2a177f60c1e88bf to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/6765a71eb8794558b2a177f60c1e88bf 2024-12-09T23:57:16,144 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/422ff75ace6f4858bd7ccd1f135d4405 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/422ff75ace6f4858bd7ccd1f135d4405 2024-12-09T23:57:16,144 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/37b87d3be795411f9ca2b277f5c526c7 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/37b87d3be795411f9ca2b277f5c526c7 2024-12-09T23:57:16,144 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/34317ca0c5624706958db763e910f8a3 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/34317ca0c5624706958db763e910f8a3 2024-12-09T23:57:16,145 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/82d9ff3835774c28b2895003afc77621 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/82d9ff3835774c28b2895003afc77621 2024-12-09T23:57:16,145 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/9c07739a9df14ea48f85c5034a9aac56 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/9c07739a9df14ea48f85c5034a9aac56 2024-12-09T23:57:16,145 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/0cdb3add422045c8bd9b8197450f2ed8 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/0cdb3add422045c8bd9b8197450f2ed8 2024-12-09T23:57:16,146 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/65badc2a4c4441bab56ce32b344a214b to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/65badc2a4c4441bab56ce32b344a214b 2024-12-09T23:57:16,146 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/00743dc7048145f886e636466cde182f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/00743dc7048145f886e636466cde182f 2024-12-09T23:57:16,146 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/150e3281f14a481fb8885e7224e2bd4f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/150e3281f14a481fb8885e7224e2bd4f 2024-12-09T23:57:16,146 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/f8f11dbb1011486aa0c068cf2ae6399b to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/f8f11dbb1011486aa0c068cf2ae6399b 2024-12-09T23:57:16,146 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/1d7ac4291d624ce281a45aa2a1a34e76 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/1d7ac4291d624ce281a45aa2a1a34e76 2024-12-09T23:57:16,147 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/d67e2984e27b4b878ec4a64a77dd1ad3 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/d67e2984e27b4b878ec4a64a77dd1ad3 2024-12-09T23:57:16,148 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/b2d7abf442924cad8e32688a1dee6401 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/b2d7abf442924cad8e32688a1dee6401 2024-12-09T23:57:16,148 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/9aeabdedfd744be69105f074affa760c to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/9aeabdedfd744be69105f074affa760c 2024-12-09T23:57:16,148 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/27d9acb9c4ad47cfaa8e271712dec620 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/27d9acb9c4ad47cfaa8e271712dec620 2024-12-09T23:57:16,148 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/2ad8aac4fc2d4f3d9070e9eabdab07be to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/2ad8aac4fc2d4f3d9070e9eabdab07be 2024-12-09T23:57:16,148 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/95d8fa7d58fc442387783f6d36215758 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/95d8fa7d58fc442387783f6d36215758 2024-12-09T23:57:16,148 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/670be54c72754068aba66da9f58dafb9 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/670be54c72754068aba66da9f58dafb9 2024-12-09T23:57:16,148 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/3c822ca84e4d4272b3bbbf0e13be22e2 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/3c822ca84e4d4272b3bbbf0e13be22e2 2024-12-09T23:57:16,149 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/2fabd41320604163b5d68adc892b539c to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/2fabd41320604163b5d68adc892b539c 2024-12-09T23:57:16,150 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/d54d9e07346f48b690b8b7a597030a1f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/d54d9e07346f48b690b8b7a597030a1f 2024-12-09T23:57:16,150 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/468d68496060493482091c8a2b55cc03 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/468d68496060493482091c8a2b55cc03 2024-12-09T23:57:16,150 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/97bf413680a94a15b22dcf3c0c4cc391 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/97bf413680a94a15b22dcf3c0c4cc391 2024-12-09T23:57:16,154 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/recovered.edits/1383.seqid, newMaxSeqId=1383, maxSeqId=1 2024-12-09T23:57:16,155 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. 2024-12-09T23:57:16,156 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for c1cc0773bc61ee89c3c2791bfea1e203: Waiting for close lock at 1733788634808Running coprocessor pre-close hooks at 1733788634808Disabling compacts and flushes for region at 1733788634808Disabling writes for close at 1733788634809 (+1 ms)Obtaining lock to block concurrent updates at 1733788634809Preparing flush snapshotting stores in c1cc0773bc61ee89c3c2791bfea1e203 at 1733788634809Finished memstore snapshotting TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1733788634810 (+1 ms)Flushing stores of TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203. at 1733788634811 (+1 ms)Flushing c1cc0773bc61ee89c3c2791bfea1e203/A: creating writer at 1733788634811Flushing c1cc0773bc61ee89c3c2791bfea1e203/A: appending metadata at 1733788634815 (+4 ms)Flushing c1cc0773bc61ee89c3c2791bfea1e203/A: closing flushed file at 1733788634815Flushing c1cc0773bc61ee89c3c2791bfea1e203/B: creating writer at 1733788635232 (+417 ms)Flushing c1cc0773bc61ee89c3c2791bfea1e203/B: appending metadata at 1733788635235 (+3 ms)Flushing c1cc0773bc61ee89c3c2791bfea1e203/B: closing flushed file at 1733788635235Flushing c1cc0773bc61ee89c3c2791bfea1e203/C: creating writer at 1733788635650 (+415 ms)Flushing c1cc0773bc61ee89c3c2791bfea1e203/C: appending metadata at 1733788635653 (+3 ms)Flushing c1cc0773bc61ee89c3c2791bfea1e203/C: closing flushed file at 1733788635653Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a5472b9: reopening flushed file at 1733788636069 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@782db0cc: reopening flushed file at 1733788636075 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b60a027: reopening flushed file at 1733788636082 (+7 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for c1cc0773bc61ee89c3c2791bfea1e203 in 1279ms, sequenceid=1380, compaction requested=false at 1733788636089 (+7 ms)Writing region close event to WAL at 1733788636151 (+62 ms)Running coprocessor post-close hooks at 1733788636155 (+4 ms)Closed at 1733788636155 2024-12-09T23:57:16,158 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:57:16,158 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=c1cc0773bc61ee89c3c2791bfea1e203, regionState=CLOSED 2024-12-09T23:57:16,160 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 because future has completed 2024-12-09T23:57:16,162 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-09T23:57:16,162 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure c1cc0773bc61ee89c3c2791bfea1e203, server=2907d75fbb3e,37627,1733788599837 in 1.5140 sec 2024-12-09T23:57:16,164 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-09T23:57:16,164 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1cc0773bc61ee89c3c2791bfea1e203, UNASSIGN in 1.5200 sec 2024-12-09T23:57:16,167 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-09T23:57:16,167 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5270 sec 2024-12-09T23:57:16,168 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733788636168"}]},"ts":"1733788636168"} 2024-12-09T23:57:16,169 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T23:57:16,169 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T23:57:16,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5400 sec 2024-12-09T23:57:16,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T23:57:16,784 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:57:16,789 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-09T23:57:16,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T23:57:16,795 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T23:57:16,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-09T23:57:16,797 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T23:57:16,801 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:57:16,806 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A, FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B, FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C, FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/recovered.edits] 2024-12-09T23:57:16,812 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/213dcfc5170c4853b586d8105400b695 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/213dcfc5170c4853b586d8105400b695 2024-12-09T23:57:16,812 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/af29573d693f4f9397632b9835f4e86e to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/A/af29573d693f4f9397632b9835f4e86e 2024-12-09T23:57:16,817 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/5f2ae540c5484c9495db6d8435cc3379 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/5f2ae540c5484c9495db6d8435cc3379 2024-12-09T23:57:16,817 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/e5c9ffbe1b6a4bff9800fcaac2c7daad to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/B/e5c9ffbe1b6a4bff9800fcaac2c7daad 2024-12-09T23:57:16,823 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/499b65ea6f8c481286f137266a4b7f24 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/499b65ea6f8c481286f137266a4b7f24 2024-12-09T23:57:16,823 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/81ec5c6145a343f69e0c217605be9ed9 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/C/81ec5c6145a343f69e0c217605be9ed9 2024-12-09T23:57:16,829 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/recovered.edits/1383.seqid to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203/recovered.edits/1383.seqid 2024-12-09T23:57:16,830 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/c1cc0773bc61ee89c3c2791bfea1e203 2024-12-09T23:57:16,830 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T23:57:16,836 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T23:57:16,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-09T23:57:16,846 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T23:57:16,876 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T23:57:16,877 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T23:57:16,877 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-09T23:57:16,878 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733788636877"}]},"ts":"9223372036854775807"} 2024-12-09T23:57:16,881 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T23:57:16,882 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c1cc0773bc61ee89c3c2791bfea1e203, NAME => 'TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T23:57:16,882 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T23:57:16,882 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733788636882"}]},"ts":"9223372036854775807"} 2024-12-09T23:57:16,885 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T23:57:16,886 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T23:57:16,888 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 96 msec 2024-12-09T23:57:16,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-09T23:57:16,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T23:57:16,904 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:57:16,915 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=254 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;2907d75fbb3e:37627-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1827791340_22 at /127.0.0.1:34300 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=333 (was 305) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4115 (was 4686) 2024-12-09T23:57:16,925 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=254, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=333, ProcessCount=11, AvailableMemoryMB=4115 2024-12-09T23:57:16,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T23:57:16,927 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T23:57:16,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T23:57:16,929 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T23:57:16,929 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:16,929 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-12-09T23:57:16,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-09T23:57:16,930 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T23:57:16,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741939_1115 (size=1061) 2024-12-09T23:57:17,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-09T23:57:17,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-09T23:57:17,341 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e 2024-12-09T23:57:17,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741940_1116 (size=53) 2024-12-09T23:57:17,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-09T23:57:17,755 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:57:17,755 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a71a18ca5bdcebe10547b57d8dd2e16f, disabling compactions & flushes 2024-12-09T23:57:17,755 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:17,755 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:17,755 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. after waiting 0 ms 2024-12-09T23:57:17,755 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:17,755 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:17,755 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a71a18ca5bdcebe10547b57d8dd2e16f: Waiting for close lock at 1733788637755Disabling compacts and flushes for region at 1733788637755Disabling writes for close at 1733788637755Writing region close event to WAL at 1733788637755Closed at 1733788637755 2024-12-09T23:57:17,758 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T23:57:17,760 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733788637759"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733788637759"}]},"ts":"1733788637759"} 2024-12-09T23:57:17,763 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T23:57:17,764 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T23:57:17,765 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733788637764"}]},"ts":"1733788637764"} 2024-12-09T23:57:17,766 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T23:57:17,766 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71a18ca5bdcebe10547b57d8dd2e16f, ASSIGN}] 2024-12-09T23:57:17,767 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71a18ca5bdcebe10547b57d8dd2e16f, ASSIGN 2024-12-09T23:57:17,768 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71a18ca5bdcebe10547b57d8dd2e16f, ASSIGN; state=OFFLINE, location=2907d75fbb3e,37627,1733788599837; forceNewPlan=false, retain=false 2024-12-09T23:57:17,919 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=a71a18ca5bdcebe10547b57d8dd2e16f, regionState=OPENING, regionLocation=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:17,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71a18ca5bdcebe10547b57d8dd2e16f, ASSIGN because future has completed 2024-12-09T23:57:17,922 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837}] 2024-12-09T23:57:18,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-09T23:57:18,085 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:18,085 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} 2024-12-09T23:57:18,086 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:18,086 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:57:18,087 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:18,087 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:18,088 INFO [StoreOpener-a71a18ca5bdcebe10547b57d8dd2e16f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:18,089 INFO [StoreOpener-a71a18ca5bdcebe10547b57d8dd2e16f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T23:57:18,090 INFO [StoreOpener-a71a18ca5bdcebe10547b57d8dd2e16f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a71a18ca5bdcebe10547b57d8dd2e16f columnFamilyName A 2024-12-09T23:57:18,090 DEBUG [StoreOpener-a71a18ca5bdcebe10547b57d8dd2e16f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:18,090 INFO [StoreOpener-a71a18ca5bdcebe10547b57d8dd2e16f-1 {}] regionserver.HStore(327): Store=a71a18ca5bdcebe10547b57d8dd2e16f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:57:18,091 INFO [StoreOpener-a71a18ca5bdcebe10547b57d8dd2e16f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:18,092 INFO [StoreOpener-a71a18ca5bdcebe10547b57d8dd2e16f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T23:57:18,092 INFO [StoreOpener-a71a18ca5bdcebe10547b57d8dd2e16f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a71a18ca5bdcebe10547b57d8dd2e16f columnFamilyName B 2024-12-09T23:57:18,092 DEBUG [StoreOpener-a71a18ca5bdcebe10547b57d8dd2e16f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:18,092 INFO [StoreOpener-a71a18ca5bdcebe10547b57d8dd2e16f-1 {}] regionserver.HStore(327): Store=a71a18ca5bdcebe10547b57d8dd2e16f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:57:18,092 INFO [StoreOpener-a71a18ca5bdcebe10547b57d8dd2e16f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:18,093 INFO [StoreOpener-a71a18ca5bdcebe10547b57d8dd2e16f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T23:57:18,093 INFO [StoreOpener-a71a18ca5bdcebe10547b57d8dd2e16f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a71a18ca5bdcebe10547b57d8dd2e16f columnFamilyName C 2024-12-09T23:57:18,093 DEBUG [StoreOpener-a71a18ca5bdcebe10547b57d8dd2e16f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:18,094 INFO [StoreOpener-a71a18ca5bdcebe10547b57d8dd2e16f-1 {}] regionserver.HStore(327): Store=a71a18ca5bdcebe10547b57d8dd2e16f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:57:18,094 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:18,094 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:18,094 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:18,095 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:18,096 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:18,096 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:18,096 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T23:57:18,097 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:18,099 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T23:57:18,100 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened a71a18ca5bdcebe10547b57d8dd2e16f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67026655, jitterRate=-0.001225009560585022}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T23:57:18,100 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:18,101 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for a71a18ca5bdcebe10547b57d8dd2e16f: Running coprocessor pre-open hook at 1733788638087Writing region info on filesystem at 1733788638087Initializing all the Stores at 1733788638088 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788638088Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788638088Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788638088Cleaning up temporary data from old regions at 1733788638096 (+8 ms)Running coprocessor post-open hooks at 1733788638100 (+4 ms)Region opened successfully at 1733788638101 (+1 ms) 2024-12-09T23:57:18,101 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., pid=32, masterSystemTime=1733788638075 2024-12-09T23:57:18,103 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:18,103 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:18,104 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=a71a18ca5bdcebe10547b57d8dd2e16f, regionState=OPEN, openSeqNum=2, regionLocation=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:18,105 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 because future has completed 2024-12-09T23:57:18,108 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-09T23:57:18,108 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 in 184 msec 2024-12-09T23:57:18,111 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-12-09T23:57:18,111 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71a18ca5bdcebe10547b57d8dd2e16f, ASSIGN in 342 msec 2024-12-09T23:57:18,111 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T23:57:18,112 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733788638111"}]},"ts":"1733788638111"} 2024-12-09T23:57:18,114 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T23:57:18,115 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T23:57:18,116 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1880 sec 2024-12-09T23:57:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-09T23:57:19,075 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:57:19,077 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b4db6c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:19,078 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:57:19,078 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:57:19,082 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:57:19,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:57:19,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:57:19,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e09993, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:19,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:57:19,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:57:19,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:19,088 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51628, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:57:19,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14ab66bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:19,089 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:57:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:57:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:57:19,093 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:57:19,094 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T23:57:19,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 2907d75fbb3e,45191,1733788599119 2024-12-09T23:57:19,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3af118e7 2024-12-09T23:57:19,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T23:57:19,096 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51642, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T23:57:19,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T23:57:19,101 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T23:57:19,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-09T23:57:19,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741941_1117 (size=1097) 2024-12-09T23:57:19,525 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-09T23:57:19,525 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-09T23:57:19,533 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T23:57:19,540 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71a18ca5bdcebe10547b57d8dd2e16f, REOPEN/MOVE}] 2024-12-09T23:57:19,541 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71a18ca5bdcebe10547b57d8dd2e16f, REOPEN/MOVE 2024-12-09T23:57:19,542 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=a71a18ca5bdcebe10547b57d8dd2e16f, regionState=CLOSING, regionLocation=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:19,543 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71a18ca5bdcebe10547b57d8dd2e16f, REOPEN/MOVE because future has completed 2024-12-09T23:57:19,544 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T23:57:19,544 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837}] 2024-12-09T23:57:19,698 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:19,698 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T23:57:19,698 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing a71a18ca5bdcebe10547b57d8dd2e16f, disabling compactions & flushes 2024-12-09T23:57:19,698 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:19,698 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:19,698 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. after waiting 0 ms 2024-12-09T23:57:19,698 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:19,746 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T23:57:19,747 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:19,748 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for a71a18ca5bdcebe10547b57d8dd2e16f: Waiting for close lock at 1733788639698Running coprocessor pre-close hooks at 1733788639698Disabling compacts and flushes for region at 1733788639698Disabling writes for close at 1733788639698Writing region close event to WAL at 1733788639699 (+1 ms)Running coprocessor post-close hooks at 1733788639747 (+48 ms)Closed at 1733788639747 2024-12-09T23:57:19,748 WARN [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: a71a18ca5bdcebe10547b57d8dd2e16f to self. 2024-12-09T23:57:19,753 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:19,754 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=a71a18ca5bdcebe10547b57d8dd2e16f, regionState=CLOSED 2024-12-09T23:57:19,756 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 because future has completed 2024-12-09T23:57:19,760 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-12-09T23:57:19,760 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 in 214 msec 2024-12-09T23:57:19,761 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71a18ca5bdcebe10547b57d8dd2e16f, REOPEN/MOVE; state=CLOSED, location=2907d75fbb3e,37627,1733788599837; forceNewPlan=false, retain=true 2024-12-09T23:57:19,912 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=a71a18ca5bdcebe10547b57d8dd2e16f, regionState=OPENING, regionLocation=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:19,917 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71a18ca5bdcebe10547b57d8dd2e16f, REOPEN/MOVE because future has completed 2024-12-09T23:57:19,918 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837}] 2024-12-09T23:57:20,079 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:20,080 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} 2024-12-09T23:57:20,081 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:20,081 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:57:20,081 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:20,081 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:20,089 INFO [StoreOpener-a71a18ca5bdcebe10547b57d8dd2e16f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:20,091 INFO [StoreOpener-a71a18ca5bdcebe10547b57d8dd2e16f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T23:57:20,099 INFO [StoreOpener-a71a18ca5bdcebe10547b57d8dd2e16f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a71a18ca5bdcebe10547b57d8dd2e16f columnFamilyName A 2024-12-09T23:57:20,102 DEBUG [StoreOpener-a71a18ca5bdcebe10547b57d8dd2e16f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:20,102 INFO [StoreOpener-a71a18ca5bdcebe10547b57d8dd2e16f-1 {}] regionserver.HStore(327): Store=a71a18ca5bdcebe10547b57d8dd2e16f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:57:20,103 INFO [StoreOpener-a71a18ca5bdcebe10547b57d8dd2e16f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:20,103 INFO [StoreOpener-a71a18ca5bdcebe10547b57d8dd2e16f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T23:57:20,104 INFO [StoreOpener-a71a18ca5bdcebe10547b57d8dd2e16f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a71a18ca5bdcebe10547b57d8dd2e16f columnFamilyName B 2024-12-09T23:57:20,104 DEBUG [StoreOpener-a71a18ca5bdcebe10547b57d8dd2e16f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:20,104 INFO [StoreOpener-a71a18ca5bdcebe10547b57d8dd2e16f-1 {}] regionserver.HStore(327): Store=a71a18ca5bdcebe10547b57d8dd2e16f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:57:20,104 INFO [StoreOpener-a71a18ca5bdcebe10547b57d8dd2e16f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:20,105 INFO [StoreOpener-a71a18ca5bdcebe10547b57d8dd2e16f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T23:57:20,105 INFO [StoreOpener-a71a18ca5bdcebe10547b57d8dd2e16f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a71a18ca5bdcebe10547b57d8dd2e16f columnFamilyName C 2024-12-09T23:57:20,105 DEBUG [StoreOpener-a71a18ca5bdcebe10547b57d8dd2e16f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:20,105 INFO [StoreOpener-a71a18ca5bdcebe10547b57d8dd2e16f-1 {}] regionserver.HStore(327): Store=a71a18ca5bdcebe10547b57d8dd2e16f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:57:20,105 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:20,105 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:20,106 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:20,107 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:20,108 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:20,108 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:20,109 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T23:57:20,110 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:20,110 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened a71a18ca5bdcebe10547b57d8dd2e16f; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74536854, jitterRate=0.11068567633628845}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T23:57:20,111 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:20,112 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for a71a18ca5bdcebe10547b57d8dd2e16f: Running coprocessor pre-open hook at 1733788640081Writing region info on filesystem at 1733788640081Initializing all the Stores at 1733788640084 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788640084Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788640084Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788640084Cleaning up temporary data from old regions at 1733788640108 (+24 ms)Running coprocessor post-open hooks at 1733788640111 (+3 ms)Region opened successfully at 1733788640112 (+1 ms) 2024-12-09T23:57:20,113 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., pid=37, masterSystemTime=1733788640072 2024-12-09T23:57:20,115 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:20,115 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:20,115 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=a71a18ca5bdcebe10547b57d8dd2e16f, regionState=OPEN, openSeqNum=5, regionLocation=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:20,117 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 because future has completed 2024-12-09T23:57:20,120 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-12-09T23:57:20,120 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 in 200 msec 2024-12-09T23:57:20,122 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-12-09T23:57:20,122 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71a18ca5bdcebe10547b57d8dd2e16f, REOPEN/MOVE in 580 msec 2024-12-09T23:57:20,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-12-09T23:57:20,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 591 msec 2024-12-09T23:57:20,129 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.0230 sec 2024-12-09T23:57:20,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-12-09T23:57:20,130 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:57:20,131 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681c9b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:20,131 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:57:20,131 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:57:20,132 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:57:20,132 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:57:20,132 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:57:20,132 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a3dedce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:20,133 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:57:20,133 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:57:20,133 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:20,134 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51662, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:57:20,134 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d084b4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:20,135 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51d91e8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:20,135 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:57:20,135 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:57:20,136 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:57:20,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:57:20,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:57:20,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1713fe00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:20,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:57:20,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:57:20,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:20,138 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51684, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:57:20,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@458a274a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:20,139 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:20,139 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:57:20,139 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:57:20,140 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:57:20,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:57:20,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:57:20,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45f92aaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:20,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:57:20,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:57:20,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:20,142 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51702, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:57:20,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@395da4bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:20,143 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b75e2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:20,143 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:57:20,143 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:57:20,144 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:57:20,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:57:20,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:57:20,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3485eca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:20,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:57:20,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:57:20,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:20,145 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51710, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:57:20,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a9d5fc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:20,146 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3f6eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:20,146 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:57:20,146 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:57:20,147 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:57:20,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:57:20,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:57:20,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f5a419, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:20,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:57:20,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:57:20,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:20,148 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51742, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:57:20,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@298a487c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:20,149 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b6e690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:20,149 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:57:20,149 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:57:20,150 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:57:20,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:57:20,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:57:20,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3902ec9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:20,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:57:20,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:57:20,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:20,152 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51760, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:57:20,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd91a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:20,153 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd05ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:20,153 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:57:20,153 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:57:20,154 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:57:20,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:57:20,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:57:20,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fc3f181, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:20,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:57:20,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:57:20,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:20,155 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51782, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:57:20,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dc1f805, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:20,156 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dff183a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:20,156 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:57:20,156 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:57:20,157 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:57:20,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:57:20,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:57:20,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51a6e16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:20,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:57:20,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:57:20,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:20,159 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51792, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:57:20,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e5bca7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:20,160 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2874ba29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:20,160 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:57:20,160 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:57:20,161 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:57:20,161 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:57:20,161 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:57:20,161 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30ab95a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:20,161 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:57:20,161 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:57:20,161 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:20,162 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51812, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:57:20,163 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15684a5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:20,164 DEBUG [Thread-603 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:57:20,164 DEBUG [Thread-608 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:57:20,164 DEBUG [Thread-605 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:57:20,164 DEBUG [Thread-612 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:57:20,164 DEBUG [Thread-601 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:57:20,165 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:57:20,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:57:20,166 DEBUG [Thread-597 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:57:20,166 DEBUG [Thread-599 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:57:20,166 DEBUG [Thread-614 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:57:20,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:57:20,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:57:20,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:57:20,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:57:20,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:57:20,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:57:20,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:57:20,168 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42758, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:57:20,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:57:20,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:57:20,168 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:57:20,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-12-09T23:57:20,169 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:57:20,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:57:20,169 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:57:20,169 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:57:20,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:57:20,170 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:57:20,170 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42802, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:57:20,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:57:20,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:57:20,171 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42780, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:57:20,171 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:57:20,171 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42810, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:57:20,171 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:57:20,172 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:57:20,172 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42836, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:57:20,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5] 2024-12-09T23:57:20,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5] 2024-12-09T23:57:20,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5] 2024-12-09T23:57:20,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-09T23:57:20,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5] 2024-12-09T23:57:20,177 DEBUG [Thread-610 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:57:20,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5] 2024-12-09T23:57:20,177 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:57:20,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:57:20,179 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42838, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:57:20,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5] 2024-12-09T23:57:20,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5] 2024-12-09T23:57:20,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5] 2024-12-09T23:57:20,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5] 2024-12-09T23:57:20,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:20,189 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a71a18ca5bdcebe10547b57d8dd2e16f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T23:57:20,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=A 2024-12-09T23:57:20,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:20,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=B 2024-12-09T23:57:20,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:20,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=C 2024-12-09T23:57:20,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:20,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733788640208, max timestamp=1733788640224Num uniques 859; 2024-12-09T23:57:20,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733788640208, max timestamp=1733788640224Num uniques 859; 2024-12-09T23:57:20,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-09T23:57:20,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-09T23:57:20,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f81ced5ba68c4913a5049dcb5251c405_a71a18ca5bdcebe10547b57d8dd2e16f is 50, key is test_row_0/A:col10/1733788640180/Put/seqid=0 2024-12-09T23:57:20,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733788640208, max timestamp=1733788640224Num uniques 859; 2024-12-09T23:57:20,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-09T23:57:20,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741942_1118 (size=14594) 2024-12-09T23:57:20,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:20,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42810 deadline: 1733788700263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:20,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:20,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42764 deadline: 1733788700265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:20,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:20,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:20,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1733788700269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:20,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:20,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:20,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:20,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42836 deadline: 1733788700272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:20,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:20,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788700272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:20,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:20,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:20,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-09T23:57:20,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-09T23:57:20,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:20,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:20,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:20,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:20,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:20,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:20,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:20,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:20,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1733788700383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:20,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:20,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:20,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42764 deadline: 1733788700383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:20,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:20,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:20,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788700383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:20,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:20,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:20,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42836 deadline: 1733788700383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:20,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:20,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42810 deadline: 1733788700382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:20,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:20,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-09T23:57:20,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:20,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:20,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:20,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:20,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:20,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-09T23:57:20,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:20,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:20,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1733788700593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:20,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42836 deadline: 1733788700593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:20,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:20,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:20,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42810 deadline: 1733788700593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:20,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788700593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:20,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:20,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:20,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42764 deadline: 1733788700593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:20,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:20,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:20,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:20,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:20,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-09T23:57:20,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:20,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:20,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:20,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:20,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:20,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:20,646 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:20,651 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f81ced5ba68c4913a5049dcb5251c405_a71a18ca5bdcebe10547b57d8dd2e16f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f81ced5ba68c4913a5049dcb5251c405_a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:20,653 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/00bddf5f99694eb0930998dd074bc90b, store: [table=TestAcidGuarantees family=A region=a71a18ca5bdcebe10547b57d8dd2e16f] 2024-12-09T23:57:20,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/00bddf5f99694eb0930998dd074bc90b is 175, key is test_row_0/A:col10/1733788640180/Put/seqid=0 2024-12-09T23:57:20,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741943_1119 (size=39549) 2024-12-09T23:57:20,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-09T23:57:20,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:20,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:20,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:20,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:20,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:20,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:20,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-09T23:57:20,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42836 deadline: 1733788700913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:20,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:20,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1733788700913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:20,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:20,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:20,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42810 deadline: 1733788700913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:20,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:20,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:20,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788700913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:20,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:20,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:20,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42764 deadline: 1733788700914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:20,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:20,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:20,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-09T23:57:20,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:20,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:20,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:20,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:20,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:20,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:21,074 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/00bddf5f99694eb0930998dd074bc90b 2024-12-09T23:57:21,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-09T23:57:21,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:21,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:21,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:21,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:21,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:21,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:21,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/bf45d9eb798e450cb8dd58ea9721390b is 50, key is test_row_0/B:col10/1733788640180/Put/seqid=0 2024-12-09T23:57:21,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741944_1120 (size=12001) 2024-12-09T23:57:21,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/bf45d9eb798e450cb8dd58ea9721390b 2024-12-09T23:57:21,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/9d73cdc7408449d28210569edda95ad4 is 50, key is test_row_0/C:col10/1733788640180/Put/seqid=0 2024-12-09T23:57:21,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741945_1121 (size=12001) 2024-12-09T23:57:21,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-09T23:57:21,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:21,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:21,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:21,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:21,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:21,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:21,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-09T23:57:21,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-09T23:57:21,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:21,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:21,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:21,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:21,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:21,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:21,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:21,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42836 deadline: 1733788701422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:21,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:21,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:21,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:21,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1733788701422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:21,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:21,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:21,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:21,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:21,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42810 deadline: 1733788701423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:21,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:21,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:21,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:21,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:21,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:21,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42764 deadline: 1733788701423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:21,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:21,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:21,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:21,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:21,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788701433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:21,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/9d73cdc7408449d28210569edda95ad4 2024-12-09T23:57:21,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-09T23:57:21,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:21,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:21,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:21,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:21,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:21,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:21,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/00bddf5f99694eb0930998dd074bc90b as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/00bddf5f99694eb0930998dd074bc90b 2024-12-09T23:57:21,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/00bddf5f99694eb0930998dd074bc90b, entries=200, sequenceid=14, filesize=38.6 K 2024-12-09T23:57:21,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/bf45d9eb798e450cb8dd58ea9721390b as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/bf45d9eb798e450cb8dd58ea9721390b 2024-12-09T23:57:21,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/bf45d9eb798e450cb8dd58ea9721390b, entries=150, sequenceid=14, filesize=11.7 K 2024-12-09T23:57:21,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/9d73cdc7408449d28210569edda95ad4 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/9d73cdc7408449d28210569edda95ad4 2024-12-09T23:57:21,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/9d73cdc7408449d28210569edda95ad4, entries=150, sequenceid=14, filesize=11.7 K 2024-12-09T23:57:21,594 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for a71a18ca5bdcebe10547b57d8dd2e16f in 1405ms, sequenceid=14, compaction requested=false 2024-12-09T23:57:21,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a71a18ca5bdcebe10547b57d8dd2e16f: 2024-12-09T23:57:21,687 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T23:57:21,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-09T23:57:21,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:21,711 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing a71a18ca5bdcebe10547b57d8dd2e16f 3/3 column families, dataSize=140.89 KB heapSize=386.16 KB 2024-12-09T23:57:21,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=A 2024-12-09T23:57:21,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:57:21,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=B 2024-12-09T23:57:21,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:57:21,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=C 2024-12-09T23:57:21,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:57:21,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209256753eea19d44d79889e9a8eb0c7015_a71a18ca5bdcebe10547b57d8dd2e16f is 50, key is test_row_0/A:col10/1733788640264/Put/seqid=0 2024-12-09T23:57:21,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741946_1122 (size=12154) 2024-12-09T23:57:21,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:21,745 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209256753eea19d44d79889e9a8eb0c7015_a71a18ca5bdcebe10547b57d8dd2e16f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209256753eea19d44d79889e9a8eb0c7015_a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:21,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/52ee60f59e134e279a554f58c429422c, store: [table=TestAcidGuarantees family=A region=a71a18ca5bdcebe10547b57d8dd2e16f] 2024-12-09T23:57:21,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/52ee60f59e134e279a554f58c429422c is 175, key is test_row_0/A:col10/1733788640264/Put/seqid=0 2024-12-09T23:57:21,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741947_1123 (size=30955) 2024-12-09T23:57:22,153 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=48, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/52ee60f59e134e279a554f58c429422c 2024-12-09T23:57:22,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/34e753f926594af39f8d05c11e4d11c4 is 50, key is test_row_0/B:col10/1733788640264/Put/seqid=0 2024-12-09T23:57:22,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741948_1124 (size=12001) 2024-12-09T23:57:22,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-09T23:57:22,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:22,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:22,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1733788702444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:22,446 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:22,446 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:22,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42836 deadline: 1733788702445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:22,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42810 deadline: 1733788702445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:22,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:22,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:22,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:22,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:22,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:22,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:22,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42764 deadline: 1733788702446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:22,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:22,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:22,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:22,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788702453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:22,457 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_2' in a71a18ca5bdcebe10547b57d8dd2e16f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:57:22,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:22,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:22,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:22,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1733788702553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:22,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42836 deadline: 1733788702553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:22,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:22,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:22,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:22,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:22,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42810 deadline: 1733788702553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:22,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:22,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:22,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:22,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:22,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:22,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42764 deadline: 1733788702563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:22,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:22,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:22,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:22,578 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/34e753f926594af39f8d05c11e4d11c4 2024-12-09T23:57:22,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/9ef085b8d190434dbe7362d8cc204a7b is 50, key is test_row_0/C:col10/1733788640264/Put/seqid=0 2024-12-09T23:57:22,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741949_1125 (size=12001) 2024-12-09T23:57:22,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1733788702763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:22,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:22,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42836 deadline: 1733788702763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:22,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:22,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:22,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:22,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:22,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:22,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42810 deadline: 1733788702763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:22,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:22,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:22,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:22,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42764 deadline: 1733788702773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:22,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:22,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:22,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:22,991 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/9ef085b8d190434dbe7362d8cc204a7b 2024-12-09T23:57:22,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/52ee60f59e134e279a554f58c429422c as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/52ee60f59e134e279a554f58c429422c 2024-12-09T23:57:23,001 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/52ee60f59e134e279a554f58c429422c, entries=150, sequenceid=48, filesize=30.2 K 2024-12-09T23:57:23,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/34e753f926594af39f8d05c11e4d11c4 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/34e753f926594af39f8d05c11e4d11c4 2024-12-09T23:57:23,007 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/34e753f926594af39f8d05c11e4d11c4, entries=150, sequenceid=48, filesize=11.7 K 2024-12-09T23:57:23,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/9ef085b8d190434dbe7362d8cc204a7b as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/9ef085b8d190434dbe7362d8cc204a7b 2024-12-09T23:57:23,016 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/9ef085b8d190434dbe7362d8cc204a7b, entries=150, sequenceid=48, filesize=11.7 K 2024-12-09T23:57:23,018 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~386.11 KB/395376, currentSize=53.67 KB/54960 for a71a18ca5bdcebe10547b57d8dd2e16f in 1306ms, sequenceid=48, compaction requested=false 2024-12-09T23:57:23,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for a71a18ca5bdcebe10547b57d8dd2e16f: 2024-12-09T23:57:23,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:23,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-12-09T23:57:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-12-09T23:57:23,024 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-12-09T23:57:23,024 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8480 sec 2024-12-09T23:57:23,026 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 2.8600 sec 2024-12-09T23:57:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:23,074 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a71a18ca5bdcebe10547b57d8dd2e16f 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T23:57:23,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=A 2024-12-09T23:57:23,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:23,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=B 2024-12-09T23:57:23,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:23,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=C 2024-12-09T23:57:23,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:23,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209b7d585851d194f0e996dd3db91c7d55b_a71a18ca5bdcebe10547b57d8dd2e16f is 50, key is test_row_0/A:col10/1733788643073/Put/seqid=0 2024-12-09T23:57:23,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741950_1126 (size=14594) 2024-12-09T23:57:23,091 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:23,096 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209b7d585851d194f0e996dd3db91c7d55b_a71a18ca5bdcebe10547b57d8dd2e16f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b7d585851d194f0e996dd3db91c7d55b_a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:23,097 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/8569d0ba2e9c40f0b5400accf3f2ae50, store: [table=TestAcidGuarantees family=A region=a71a18ca5bdcebe10547b57d8dd2e16f] 2024-12-09T23:57:23,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/8569d0ba2e9c40f0b5400accf3f2ae50 is 175, key is test_row_0/A:col10/1733788643073/Put/seqid=0 2024-12-09T23:57:23,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:23,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:23,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42810 deadline: 1733788703095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:23,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42836 deadline: 1733788703096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:23,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:23,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:23,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:23,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:23,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:23,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:23,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741951_1127 (size=39549) 2024-12-09T23:57:23,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:23,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42764 deadline: 1733788703101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:23,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1733788703101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:23,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42836 deadline: 1733788703203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:23,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:23,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:23,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:23,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42810 deadline: 1733788703212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:23,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42764 deadline: 1733788703213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:23,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:23,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1733788703213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:23,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:23,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:23,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:23,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:23,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:23,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:23,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:23,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:23,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:23,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42836 deadline: 1733788703412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:23,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:23,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:23,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:23,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:23,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42810 deadline: 1733788703423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:23,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:23,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:23,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42764 deadline: 1733788703423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:23,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:23,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:23,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:23,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:23,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:23,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:23,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1733788703423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:23,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:23,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:23,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:23,504 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/8569d0ba2e9c40f0b5400accf3f2ae50 2024-12-09T23:57:23,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/2b3eca773ce74b388677c014693d0589 is 50, key is test_row_0/B:col10/1733788643073/Put/seqid=0 2024-12-09T23:57:23,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741952_1128 (size=12001) 2024-12-09T23:57:23,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42836 deadline: 1733788703722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:23,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:23,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:23,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:23,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42810 deadline: 1733788703732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:23,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42764 deadline: 1733788703732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:23,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1733788703733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:23,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/2b3eca773ce74b388677c014693d0589 2024-12-09T23:57:23,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/d9e522cfb45542e2a953d45a0b06df8f is 50, key is test_row_0/C:col10/1733788643073/Put/seqid=0 2024-12-09T23:57:23,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741953_1129 (size=12001) 2024-12-09T23:57:24,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:24,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42836 deadline: 1733788704232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:24,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:24,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:24,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:24,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:24,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42810 deadline: 1733788704242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:24,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:24,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:24,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:24,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:24,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42764 deadline: 1733788704243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:24,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:24,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:24,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:24,251 INFO [master/2907d75fbb3e:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-09T23:57:24,251 INFO [master/2907d75fbb3e:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-09T23:57:24,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:24,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1733788704253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:24,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:24,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:24,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:24,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-09T23:57:24,343 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:57:24,343 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:57:24,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-09T23:57:24,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-09T23:57:24,345 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:57:24,345 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:57:24,346 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:57:24,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/d9e522cfb45542e2a953d45a0b06df8f 2024-12-09T23:57:24,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/8569d0ba2e9c40f0b5400accf3f2ae50 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/8569d0ba2e9c40f0b5400accf3f2ae50 2024-12-09T23:57:24,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/8569d0ba2e9c40f0b5400accf3f2ae50, entries=200, sequenceid=62, filesize=38.6 K 2024-12-09T23:57:24,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/2b3eca773ce74b388677c014693d0589 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/2b3eca773ce74b388677c014693d0589 2024-12-09T23:57:24,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/2b3eca773ce74b388677c014693d0589, entries=150, sequenceid=62, filesize=11.7 K 2024-12-09T23:57:24,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/d9e522cfb45542e2a953d45a0b06df8f as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/d9e522cfb45542e2a953d45a0b06df8f 2024-12-09T23:57:24,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/d9e522cfb45542e2a953d45a0b06df8f, entries=150, sequenceid=62, filesize=11.7 K 2024-12-09T23:57:24,372 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for a71a18ca5bdcebe10547b57d8dd2e16f in 1298ms, sequenceid=62, compaction requested=true 2024-12-09T23:57:24,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a71a18ca5bdcebe10547b57d8dd2e16f: 2024-12-09T23:57:24,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71a18ca5bdcebe10547b57d8dd2e16f:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:57:24,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:24,372 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:57:24,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71a18ca5bdcebe10547b57d8dd2e16f:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:57:24,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:24,372 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:57:24,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71a18ca5bdcebe10547b57d8dd2e16f:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:57:24,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:57:24,374 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:57:24,374 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:57:24,374 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): a71a18ca5bdcebe10547b57d8dd2e16f/A is initiating minor compaction (all files) 2024-12-09T23:57:24,374 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): a71a18ca5bdcebe10547b57d8dd2e16f/B is initiating minor compaction (all files) 2024-12-09T23:57:24,374 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71a18ca5bdcebe10547b57d8dd2e16f/A in TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:24,374 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71a18ca5bdcebe10547b57d8dd2e16f/B in TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:24,374 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/00bddf5f99694eb0930998dd074bc90b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/52ee60f59e134e279a554f58c429422c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/8569d0ba2e9c40f0b5400accf3f2ae50] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp, totalSize=107.5 K 2024-12-09T23:57:24,374 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/bf45d9eb798e450cb8dd58ea9721390b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/34e753f926594af39f8d05c11e4d11c4, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/2b3eca773ce74b388677c014693d0589] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp, totalSize=35.2 K 2024-12-09T23:57:24,374 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:24,374 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting bf45d9eb798e450cb8dd58ea9721390b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733788640179 2024-12-09T23:57:24,374 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. files: [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/00bddf5f99694eb0930998dd074bc90b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/52ee60f59e134e279a554f58c429422c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/8569d0ba2e9c40f0b5400accf3f2ae50] 2024-12-09T23:57:24,375 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 34e753f926594af39f8d05c11e4d11c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733788640262 2024-12-09T23:57:24,375 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00bddf5f99694eb0930998dd074bc90b, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733788640179 2024-12-09T23:57:24,375 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52ee60f59e134e279a554f58c429422c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733788640262 2024-12-09T23:57:24,375 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b3eca773ce74b388677c014693d0589, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733788642444 2024-12-09T23:57:24,376 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8569d0ba2e9c40f0b5400accf3f2ae50, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733788642444 2024-12-09T23:57:24,381 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a71a18ca5bdcebe10547b57d8dd2e16f] 2024-12-09T23:57:24,387 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71a18ca5bdcebe10547b57d8dd2e16f#B#compaction#111 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:57:24,388 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/21693dea849c47269b076b380514a9c8 is 50, key is test_row_0/B:col10/1733788643073/Put/seqid=0 2024-12-09T23:57:24,391 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209c8c19d6e230142b39400ae9807e9e1a5_a71a18ca5bdcebe10547b57d8dd2e16f store=[table=TestAcidGuarantees family=A region=a71a18ca5bdcebe10547b57d8dd2e16f] 2024-12-09T23:57:24,398 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209c8c19d6e230142b39400ae9807e9e1a5_a71a18ca5bdcebe10547b57d8dd2e16f, store=[table=TestAcidGuarantees family=A region=a71a18ca5bdcebe10547b57d8dd2e16f] 2024-12-09T23:57:24,398 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c8c19d6e230142b39400ae9807e9e1a5_a71a18ca5bdcebe10547b57d8dd2e16f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a71a18ca5bdcebe10547b57d8dd2e16f] 2024-12-09T23:57:24,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741955_1131 (size=4469) 2024-12-09T23:57:24,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741954_1130 (size=12104) 2024-12-09T23:57:24,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-09T23:57:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:24,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a71a18ca5bdcebe10547b57d8dd2e16f 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T23:57:24,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=A 2024-12-09T23:57:24,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:24,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=B 2024-12-09T23:57:24,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:24,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=C 2024-12-09T23:57:24,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:24,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-09T23:57:24,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:24,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:24,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:24,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:24,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:24,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:24,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d5fcebde4e994e04a86709488fb3e901_a71a18ca5bdcebe10547b57d8dd2e16f is 50, key is test_row_0/A:col10/1733788643099/Put/seqid=0 2024-12-09T23:57:24,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741956_1132 (size=14594) 2024-12-09T23:57:24,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788704532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:24,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:24,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:24,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:24,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:24,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788704642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:24,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:24,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:24,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:24,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-09T23:57:24,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:24,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:24,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:24,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:24,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:24,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:24,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-09T23:57:24,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-09T23:57:24,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:24,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:24,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:24,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:24,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:24,851 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71a18ca5bdcebe10547b57d8dd2e16f#A#compaction#112 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:57:24,852 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/1b64e92846b840ebb974392db1ce0c98 is 175, key is test_row_0/A:col10/1733788643073/Put/seqid=0 2024-12-09T23:57:24,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:24,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788704853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:24,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:24,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:24,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:24,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741957_1133 (size=31058) 2024-12-09T23:57:24,864 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/21693dea849c47269b076b380514a9c8 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/21693dea849c47269b076b380514a9c8 2024-12-09T23:57:24,874 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/1b64e92846b840ebb974392db1ce0c98 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/1b64e92846b840ebb974392db1ce0c98 2024-12-09T23:57:24,875 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71a18ca5bdcebe10547b57d8dd2e16f/B of a71a18ca5bdcebe10547b57d8dd2e16f into 21693dea849c47269b076b380514a9c8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:24,875 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71a18ca5bdcebe10547b57d8dd2e16f: 2024-12-09T23:57:24,875 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., storeName=a71a18ca5bdcebe10547b57d8dd2e16f/B, priority=13, startTime=1733788644372; duration=0sec 2024-12-09T23:57:24,875 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:57:24,875 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71a18ca5bdcebe10547b57d8dd2e16f:B 2024-12-09T23:57:24,876 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:57:24,878 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:57:24,878 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): a71a18ca5bdcebe10547b57d8dd2e16f/C is initiating minor compaction (all files) 2024-12-09T23:57:24,878 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71a18ca5bdcebe10547b57d8dd2e16f/C in TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:24,878 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/9d73cdc7408449d28210569edda95ad4, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/9ef085b8d190434dbe7362d8cc204a7b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/d9e522cfb45542e2a953d45a0b06df8f] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp, totalSize=35.2 K 2024-12-09T23:57:24,880 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d73cdc7408449d28210569edda95ad4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733788640179 2024-12-09T23:57:24,881 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ef085b8d190434dbe7362d8cc204a7b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733788640262 2024-12-09T23:57:24,882 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting d9e522cfb45542e2a953d45a0b06df8f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733788642444 2024-12-09T23:57:24,886 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71a18ca5bdcebe10547b57d8dd2e16f/A of a71a18ca5bdcebe10547b57d8dd2e16f into 1b64e92846b840ebb974392db1ce0c98(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:24,886 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71a18ca5bdcebe10547b57d8dd2e16f: 2024-12-09T23:57:24,886 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., storeName=a71a18ca5bdcebe10547b57d8dd2e16f/A, priority=13, startTime=1733788644372; duration=0sec 2024-12-09T23:57:24,886 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:24,886 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71a18ca5bdcebe10547b57d8dd2e16f:A 2024-12-09T23:57:24,898 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71a18ca5bdcebe10547b57d8dd2e16f#C#compaction#114 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:57:24,899 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/9bbdf3bd493848d1ab59fcf87469c32c is 50, key is test_row_0/C:col10/1733788643073/Put/seqid=0 2024-12-09T23:57:24,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741958_1134 (size=12104) 2024-12-09T23:57:24,917 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:24,923 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d5fcebde4e994e04a86709488fb3e901_a71a18ca5bdcebe10547b57d8dd2e16f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d5fcebde4e994e04a86709488fb3e901_a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:24,923 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/9bbdf3bd493848d1ab59fcf87469c32c as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/9bbdf3bd493848d1ab59fcf87469c32c 2024-12-09T23:57:24,926 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/bad9ee60fcc64d219c05104c7e2099dd, store: [table=TestAcidGuarantees family=A region=a71a18ca5bdcebe10547b57d8dd2e16f] 2024-12-09T23:57:24,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/bad9ee60fcc64d219c05104c7e2099dd is 175, key is test_row_0/A:col10/1733788643099/Put/seqid=0 2024-12-09T23:57:24,931 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71a18ca5bdcebe10547b57d8dd2e16f/C of a71a18ca5bdcebe10547b57d8dd2e16f into 9bbdf3bd493848d1ab59fcf87469c32c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:24,931 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71a18ca5bdcebe10547b57d8dd2e16f: 2024-12-09T23:57:24,931 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., storeName=a71a18ca5bdcebe10547b57d8dd2e16f/C, priority=13, startTime=1733788644372; duration=0sec 2024-12-09T23:57:24,931 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:24,931 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71a18ca5bdcebe10547b57d8dd2e16f:C 2024-12-09T23:57:24,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741959_1135 (size=39549) 2024-12-09T23:57:24,937 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/bad9ee60fcc64d219c05104c7e2099dd 2024-12-09T23:57:24,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/2089556f6d3e49d49fcd36d96b9d4394 is 50, key is test_row_0/B:col10/1733788643099/Put/seqid=0 2024-12-09T23:57:24,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-09T23:57:24,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:24,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:24,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:24,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:24,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:24,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:24,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741960_1136 (size=12001) 2024-12-09T23:57:24,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-09T23:57:25,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-09T23:57:25,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:25,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:25,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:25,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:25,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:25,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:25,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:25,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788705163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:25,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:25,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:25,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:25,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:25,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42836 deadline: 1733788705253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:25,255 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_2' in a71a18ca5bdcebe10547b57d8dd2e16f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:57:25,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:25,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:25,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:25,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:25,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42810 deadline: 1733788705263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:25,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:25,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42764 deadline: 1733788705263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:25,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:25,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1733788705263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:25,264 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_1' in a71a18ca5bdcebe10547b57d8dd2e16f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:57:25,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:25,264 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_0' in a71a18ca5bdcebe10547b57d8dd2e16f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:57:25,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:25,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:25,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:25,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:25,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:25,264 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_2' in a71a18ca5bdcebe10547b57d8dd2e16f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:57:25,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-09T23:57:25,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:25,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:25,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:25,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:25,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:25,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:25,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:25,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:25,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:25,360 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/2089556f6d3e49d49fcd36d96b9d4394 2024-12-09T23:57:25,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/d2b8b3df0658468c829f4f793323dc64 is 50, key is test_row_0/C:col10/1733788643099/Put/seqid=0 2024-12-09T23:57:25,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741961_1137 (size=12001) 2024-12-09T23:57:25,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-09T23:57:25,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:25,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:25,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:25,419 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:25,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:25,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:25,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-09T23:57:25,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-09T23:57:25,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:25,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:25,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:25,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:25,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:25,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:25,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:25,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788705673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:25,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-09T23:57:25,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:25,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:25,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:25,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:25,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:25,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:25,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/d2b8b3df0658468c829f4f793323dc64 2024-12-09T23:57:25,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/bad9ee60fcc64d219c05104c7e2099dd as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/bad9ee60fcc64d219c05104c7e2099dd 2024-12-09T23:57:25,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/bad9ee60fcc64d219c05104c7e2099dd, entries=200, sequenceid=81, filesize=38.6 K 2024-12-09T23:57:25,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/2089556f6d3e49d49fcd36d96b9d4394 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/2089556f6d3e49d49fcd36d96b9d4394 2024-12-09T23:57:25,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/2089556f6d3e49d49fcd36d96b9d4394, entries=150, sequenceid=81, filesize=11.7 K 2024-12-09T23:57:25,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/d2b8b3df0658468c829f4f793323dc64 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/d2b8b3df0658468c829f4f793323dc64 2024-12-09T23:57:25,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/d2b8b3df0658468c829f4f793323dc64, entries=150, sequenceid=81, filesize=11.7 K 2024-12-09T23:57:25,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for a71a18ca5bdcebe10547b57d8dd2e16f in 1305ms, sequenceid=81, compaction requested=false 2024-12-09T23:57:25,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a71a18ca5bdcebe10547b57d8dd2e16f: 2024-12-09T23:57:25,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-09T23:57:25,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:25,878 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing a71a18ca5bdcebe10547b57d8dd2e16f 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T23:57:25,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=A 2024-12-09T23:57:25,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:25,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=B 2024-12-09T23:57:25,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:25,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=C 2024-12-09T23:57:25,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:25,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412091b41d339a676480e9e087a6ba319a674_a71a18ca5bdcebe10547b57d8dd2e16f is 50, key is test_row_0/A:col10/1733788644530/Put/seqid=0 2024-12-09T23:57:25,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741962_1138 (size=12154) 2024-12-09T23:57:25,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:25,921 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412091b41d339a676480e9e087a6ba319a674_a71a18ca5bdcebe10547b57d8dd2e16f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091b41d339a676480e9e087a6ba319a674_a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:25,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/6af81dbb28d14245899f9292311317b8, store: [table=TestAcidGuarantees family=A region=a71a18ca5bdcebe10547b57d8dd2e16f] 2024-12-09T23:57:25,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/6af81dbb28d14245899f9292311317b8 is 175, key is test_row_0/A:col10/1733788644530/Put/seqid=0 2024-12-09T23:57:25,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741963_1139 (size=30955) 2024-12-09T23:57:25,950 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/6af81dbb28d14245899f9292311317b8 2024-12-09T23:57:25,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/1eb125861cc44563abe93d4925a8d20a is 50, key is test_row_0/B:col10/1733788644530/Put/seqid=0 2024-12-09T23:57:25,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741964_1140 (size=12001) 2024-12-09T23:57:26,389 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/1eb125861cc44563abe93d4925a8d20a 2024-12-09T23:57:26,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/c9da8494ebb64820956c08bbcb29104c is 50, key is test_row_0/C:col10/1733788644530/Put/seqid=0 2024-12-09T23:57:26,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741965_1141 (size=12001) 2024-12-09T23:57:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-09T23:57:26,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:26,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:26,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788706741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:26,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:26,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:26,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:26,801 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/c9da8494ebb64820956c08bbcb29104c 2024-12-09T23:57:26,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/6af81dbb28d14245899f9292311317b8 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/6af81dbb28d14245899f9292311317b8 2024-12-09T23:57:26,812 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/6af81dbb28d14245899f9292311317b8, entries=150, sequenceid=97, filesize=30.2 K 2024-12-09T23:57:26,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/1eb125861cc44563abe93d4925a8d20a as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/1eb125861cc44563abe93d4925a8d20a 2024-12-09T23:57:26,822 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/1eb125861cc44563abe93d4925a8d20a, entries=150, sequenceid=97, filesize=11.7 K 2024-12-09T23:57:26,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/c9da8494ebb64820956c08bbcb29104c as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/c9da8494ebb64820956c08bbcb29104c 2024-12-09T23:57:26,831 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/c9da8494ebb64820956c08bbcb29104c, entries=150, sequenceid=97, filesize=11.7 K 2024-12-09T23:57:26,832 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for a71a18ca5bdcebe10547b57d8dd2e16f in 954ms, sequenceid=97, compaction requested=true 2024-12-09T23:57:26,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for a71a18ca5bdcebe10547b57d8dd2e16f: 2024-12-09T23:57:26,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:26,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-09T23:57:26,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-09T23:57:26,836 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-09T23:57:26,836 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4880 sec 2024-12-09T23:57:26,837 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.4930 sec 2024-12-09T23:57:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:26,854 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a71a18ca5bdcebe10547b57d8dd2e16f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T23:57:26,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=A 2024-12-09T23:57:26,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:26,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=B 2024-12-09T23:57:26,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:26,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=C 2024-12-09T23:57:26,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:26,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209730121f34493464382630a1d989433be_a71a18ca5bdcebe10547b57d8dd2e16f is 50, key is test_row_0/A:col10/1733788646853/Put/seqid=0 2024-12-09T23:57:26,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741966_1142 (size=14594) 2024-12-09T23:57:26,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788706896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:26,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:26,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:26,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:27,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788707003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:27,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:27,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:27,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:27,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:27,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788707213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:27,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:27,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:27,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:27,267 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:27,271 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209730121f34493464382630a1d989433be_a71a18ca5bdcebe10547b57d8dd2e16f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209730121f34493464382630a1d989433be_a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:27,273 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/827cc18d1f844488bf01c965b05f4ad8, store: [table=TestAcidGuarantees family=A region=a71a18ca5bdcebe10547b57d8dd2e16f] 2024-12-09T23:57:27,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/827cc18d1f844488bf01c965b05f4ad8 is 175, key is test_row_0/A:col10/1733788646853/Put/seqid=0 2024-12-09T23:57:27,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:27,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42810 deadline: 1733788707273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:27,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:27,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:27,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42836 deadline: 1733788707273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:27,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42764 deadline: 1733788707273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:27,278 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_2' in a71a18ca5bdcebe10547b57d8dd2e16f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:57:27,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:27,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:27,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:27,279 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_1' in a71a18ca5bdcebe10547b57d8dd2e16f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:57:27,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:27,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:27,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:27,279 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_0' in a71a18ca5bdcebe10547b57d8dd2e16f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:57:27,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:27,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:27,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:27,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:27,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1733788707283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:27,286 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_2' in a71a18ca5bdcebe10547b57d8dd2e16f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:57:27,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:27,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:27,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:27,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741967_1143 (size=39549) 2024-12-09T23:57:27,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:27,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788707523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:27,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:27,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:27,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:27,705 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/827cc18d1f844488bf01c965b05f4ad8 2024-12-09T23:57:27,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/952ed4855b7044b69fd980235fdf0fd3 is 50, key is test_row_0/B:col10/1733788646853/Put/seqid=0 2024-12-09T23:57:27,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741968_1144 (size=12001) 2024-12-09T23:57:27,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/952ed4855b7044b69fd980235fdf0fd3 2024-12-09T23:57:27,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/e8ba0aeadf3a47cbb3a0ab28a369049e is 50, key is test_row_0/C:col10/1733788646853/Put/seqid=0 2024-12-09T23:57:27,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741969_1145 (size=12001) 2024-12-09T23:57:27,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/e8ba0aeadf3a47cbb3a0ab28a369049e 2024-12-09T23:57:27,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/827cc18d1f844488bf01c965b05f4ad8 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/827cc18d1f844488bf01c965b05f4ad8 2024-12-09T23:57:27,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/827cc18d1f844488bf01c965b05f4ad8, entries=200, sequenceid=117, filesize=38.6 K 2024-12-09T23:57:27,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/952ed4855b7044b69fd980235fdf0fd3 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/952ed4855b7044b69fd980235fdf0fd3 2024-12-09T23:57:27,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/952ed4855b7044b69fd980235fdf0fd3, entries=150, sequenceid=117, filesize=11.7 K 2024-12-09T23:57:27,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/e8ba0aeadf3a47cbb3a0ab28a369049e as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/e8ba0aeadf3a47cbb3a0ab28a369049e 2024-12-09T23:57:27,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/e8ba0aeadf3a47cbb3a0ab28a369049e, entries=150, sequenceid=117, filesize=11.7 K 2024-12-09T23:57:27,767 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a71a18ca5bdcebe10547b57d8dd2e16f in 913ms, sequenceid=117, compaction requested=true 2024-12-09T23:57:27,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a71a18ca5bdcebe10547b57d8dd2e16f: 2024-12-09T23:57:27,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71a18ca5bdcebe10547b57d8dd2e16f:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:57:27,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:27,767 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T23:57:27,767 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T23:57:27,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71a18ca5bdcebe10547b57d8dd2e16f:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:57:27,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:27,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71a18ca5bdcebe10547b57d8dd2e16f:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:57:27,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:57:27,769 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141111 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T23:57:27,769 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): a71a18ca5bdcebe10547b57d8dd2e16f/A is initiating minor compaction (all files) 2024-12-09T23:57:27,769 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71a18ca5bdcebe10547b57d8dd2e16f/A in TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:27,769 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/1b64e92846b840ebb974392db1ce0c98, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/bad9ee60fcc64d219c05104c7e2099dd, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/6af81dbb28d14245899f9292311317b8, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/827cc18d1f844488bf01c965b05f4ad8] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp, totalSize=137.8 K 2024-12-09T23:57:27,769 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:27,769 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. files: [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/1b64e92846b840ebb974392db1ce0c98, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/bad9ee60fcc64d219c05104c7e2099dd, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/6af81dbb28d14245899f9292311317b8, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/827cc18d1f844488bf01c965b05f4ad8] 2024-12-09T23:57:27,770 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T23:57:27,770 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): a71a18ca5bdcebe10547b57d8dd2e16f/B is initiating minor compaction (all files) 2024-12-09T23:57:27,770 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71a18ca5bdcebe10547b57d8dd2e16f/B in TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:27,770 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/21693dea849c47269b076b380514a9c8, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/2089556f6d3e49d49fcd36d96b9d4394, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/1eb125861cc44563abe93d4925a8d20a, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/952ed4855b7044b69fd980235fdf0fd3] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp, totalSize=47.0 K 2024-12-09T23:57:27,771 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b64e92846b840ebb974392db1ce0c98, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733788642444 2024-12-09T23:57:27,771 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 21693dea849c47269b076b380514a9c8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733788642444 2024-12-09T23:57:27,771 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 2089556f6d3e49d49fcd36d96b9d4394, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733788643083 2024-12-09T23:57:27,771 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting bad9ee60fcc64d219c05104c7e2099dd, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733788643083 2024-12-09T23:57:27,772 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 1eb125861cc44563abe93d4925a8d20a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733788644510 2024-12-09T23:57:27,772 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6af81dbb28d14245899f9292311317b8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733788644510 2024-12-09T23:57:27,772 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 952ed4855b7044b69fd980235fdf0fd3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733788646729 2024-12-09T23:57:27,773 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 827cc18d1f844488bf01c965b05f4ad8, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733788646729 2024-12-09T23:57:27,781 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a71a18ca5bdcebe10547b57d8dd2e16f] 2024-12-09T23:57:27,785 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71a18ca5bdcebe10547b57d8dd2e16f#B#compaction#124 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:57:27,786 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/b11fe4f3cd78426f82d647655e9cabf8 is 50, key is test_row_0/B:col10/1733788646853/Put/seqid=0 2024-12-09T23:57:27,787 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209a8aee82c8a00458885d8f570030312bb_a71a18ca5bdcebe10547b57d8dd2e16f store=[table=TestAcidGuarantees family=A region=a71a18ca5bdcebe10547b57d8dd2e16f] 2024-12-09T23:57:27,792 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209a8aee82c8a00458885d8f570030312bb_a71a18ca5bdcebe10547b57d8dd2e16f, store=[table=TestAcidGuarantees family=A region=a71a18ca5bdcebe10547b57d8dd2e16f] 2024-12-09T23:57:27,792 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209a8aee82c8a00458885d8f570030312bb_a71a18ca5bdcebe10547b57d8dd2e16f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a71a18ca5bdcebe10547b57d8dd2e16f] 2024-12-09T23:57:27,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741970_1146 (size=12241) 2024-12-09T23:57:27,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741971_1147 (size=4469) 2024-12-09T23:57:27,799 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/b11fe4f3cd78426f82d647655e9cabf8 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/b11fe4f3cd78426f82d647655e9cabf8 2024-12-09T23:57:27,800 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71a18ca5bdcebe10547b57d8dd2e16f#A#compaction#123 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:57:27,802 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/d25b341c859547b9bb52e351f1a69f09 is 175, key is test_row_0/A:col10/1733788646853/Put/seqid=0 2024-12-09T23:57:27,805 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a71a18ca5bdcebe10547b57d8dd2e16f/B of a71a18ca5bdcebe10547b57d8dd2e16f into b11fe4f3cd78426f82d647655e9cabf8(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:27,805 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71a18ca5bdcebe10547b57d8dd2e16f: 2024-12-09T23:57:27,805 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., storeName=a71a18ca5bdcebe10547b57d8dd2e16f/B, priority=12, startTime=1733788647767; duration=0sec 2024-12-09T23:57:27,805 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:57:27,805 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71a18ca5bdcebe10547b57d8dd2e16f:B 2024-12-09T23:57:27,805 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T23:57:27,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741972_1148 (size=31195) 2024-12-09T23:57:27,810 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T23:57:27,810 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): a71a18ca5bdcebe10547b57d8dd2e16f/C is initiating minor compaction (all files) 2024-12-09T23:57:27,810 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71a18ca5bdcebe10547b57d8dd2e16f/C in TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:27,811 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/9bbdf3bd493848d1ab59fcf87469c32c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/d2b8b3df0658468c829f4f793323dc64, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/c9da8494ebb64820956c08bbcb29104c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/e8ba0aeadf3a47cbb3a0ab28a369049e] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp, totalSize=47.0 K 2024-12-09T23:57:27,811 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bbdf3bd493848d1ab59fcf87469c32c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733788642444 2024-12-09T23:57:27,812 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting d2b8b3df0658468c829f4f793323dc64, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733788643083 2024-12-09T23:57:27,812 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting c9da8494ebb64820956c08bbcb29104c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733788644510 2024-12-09T23:57:27,812 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting e8ba0aeadf3a47cbb3a0ab28a369049e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733788646729 2024-12-09T23:57:27,816 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/d25b341c859547b9bb52e351f1a69f09 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/d25b341c859547b9bb52e351f1a69f09 2024-12-09T23:57:27,825 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71a18ca5bdcebe10547b57d8dd2e16f#C#compaction#125 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:57:27,827 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/fcc07f32e5f24b33b8cb5a832b31846e is 50, key is test_row_0/C:col10/1733788646853/Put/seqid=0 2024-12-09T23:57:27,835 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a71a18ca5bdcebe10547b57d8dd2e16f/A of a71a18ca5bdcebe10547b57d8dd2e16f into d25b341c859547b9bb52e351f1a69f09(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:27,835 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71a18ca5bdcebe10547b57d8dd2e16f: 2024-12-09T23:57:27,835 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., storeName=a71a18ca5bdcebe10547b57d8dd2e16f/A, priority=12, startTime=1733788647767; duration=0sec 2024-12-09T23:57:27,835 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:27,835 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71a18ca5bdcebe10547b57d8dd2e16f:A 2024-12-09T23:57:27,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741973_1149 (size=12241) 2024-12-09T23:57:28,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:28,037 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a71a18ca5bdcebe10547b57d8dd2e16f 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T23:57:28,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=A 2024-12-09T23:57:28,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:28,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=B 2024-12-09T23:57:28,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:28,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=C 2024-12-09T23:57:28,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:28,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d7a33a2f51fb46ca9467ea82d75fa908_a71a18ca5bdcebe10547b57d8dd2e16f is 50, key is test_row_0/A:col10/1733788648035/Put/seqid=0 2024-12-09T23:57:28,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741974_1150 (size=14644) 2024-12-09T23:57:28,050 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:28,059 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d7a33a2f51fb46ca9467ea82d75fa908_a71a18ca5bdcebe10547b57d8dd2e16f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d7a33a2f51fb46ca9467ea82d75fa908_a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:28,060 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/92dc398ac9ba4bbb93cb2d4b9538ff53, store: [table=TestAcidGuarantees family=A region=a71a18ca5bdcebe10547b57d8dd2e16f] 2024-12-09T23:57:28,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/92dc398ac9ba4bbb93cb2d4b9538ff53 is 175, key is test_row_0/A:col10/1733788648035/Put/seqid=0 2024-12-09T23:57:28,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741975_1151 (size=39599) 2024-12-09T23:57:28,068 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/92dc398ac9ba4bbb93cb2d4b9538ff53 2024-12-09T23:57:28,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/3b17060f2e0942238e17b8a5bcb382df is 50, key is test_row_0/B:col10/1733788648035/Put/seqid=0 2024-12-09T23:57:28,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741976_1152 (size=12051) 2024-12-09T23:57:28,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788708105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:28,108 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:28,108 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:28,108 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:28,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:28,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788708212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:28,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:28,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:28,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:28,247 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/fcc07f32e5f24b33b8cb5a832b31846e as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/fcc07f32e5f24b33b8cb5a832b31846e 2024-12-09T23:57:28,252 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a71a18ca5bdcebe10547b57d8dd2e16f/C of a71a18ca5bdcebe10547b57d8dd2e16f into fcc07f32e5f24b33b8cb5a832b31846e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:28,252 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71a18ca5bdcebe10547b57d8dd2e16f: 2024-12-09T23:57:28,252 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., storeName=a71a18ca5bdcebe10547b57d8dd2e16f/C, priority=12, startTime=1733788647767; duration=0sec 2024-12-09T23:57:28,252 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:28,252 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71a18ca5bdcebe10547b57d8dd2e16f:C 2024-12-09T23:57:28,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:28,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788708423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:28,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/3b17060f2e0942238e17b8a5bcb382df 2024-12-09T23:57:28,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/a27a61b5e9f7463c9e7999a53dd35a83 is 50, key is test_row_0/C:col10/1733788648035/Put/seqid=0 2024-12-09T23:57:28,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741977_1153 (size=12051) 2024-12-09T23:57:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-09T23:57:28,513 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:57:28,513 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:57:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-09T23:57:28,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T23:57:28,515 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:57:28,515 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:57:28,516 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:57:28,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T23:57:28,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T23:57:28,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:28,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:28,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:28,668 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:28,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:28,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:28,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788708733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:28,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:28,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:28,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:28,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T23:57:28,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:28,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:28,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:28,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:28,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:28,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:28,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T23:57:28,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/a27a61b5e9f7463c9e7999a53dd35a83 2024-12-09T23:57:28,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/92dc398ac9ba4bbb93cb2d4b9538ff53 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/92dc398ac9ba4bbb93cb2d4b9538ff53 2024-12-09T23:57:28,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/92dc398ac9ba4bbb93cb2d4b9538ff53, entries=200, sequenceid=132, filesize=38.7 K 2024-12-09T23:57:28,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/3b17060f2e0942238e17b8a5bcb382df as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/3b17060f2e0942238e17b8a5bcb382df 2024-12-09T23:57:28,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/3b17060f2e0942238e17b8a5bcb382df, entries=150, sequenceid=132, filesize=11.8 K 2024-12-09T23:57:28,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/a27a61b5e9f7463c9e7999a53dd35a83 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/a27a61b5e9f7463c9e7999a53dd35a83 2024-12-09T23:57:28,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/a27a61b5e9f7463c9e7999a53dd35a83, entries=150, sequenceid=132, filesize=11.8 K 2024-12-09T23:57:28,928 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for a71a18ca5bdcebe10547b57d8dd2e16f in 891ms, sequenceid=132, compaction requested=false 2024-12-09T23:57:28,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a71a18ca5bdcebe10547b57d8dd2e16f: 2024-12-09T23:57:28,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T23:57:28,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:28,973 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing a71a18ca5bdcebe10547b57d8dd2e16f 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T23:57:28,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=A 2024-12-09T23:57:28,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:28,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=B 2024-12-09T23:57:28,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:28,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=C 2024-12-09T23:57:28,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:28,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412097cf7711584604ce28844dcd0cd2a76c8_a71a18ca5bdcebe10547b57d8dd2e16f is 50, key is test_row_0/A:col10/1733788648102/Put/seqid=0 2024-12-09T23:57:28,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741978_1154 (size=12304) 2024-12-09T23:57:28,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,000 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412097cf7711584604ce28844dcd0cd2a76c8_a71a18ca5bdcebe10547b57d8dd2e16f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412097cf7711584604ce28844dcd0cd2a76c8_a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:29,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/a849eac66caa445cb7926ef2e07efcf1, store: [table=TestAcidGuarantees family=A region=a71a18ca5bdcebe10547b57d8dd2e16f] 2024-12-09T23:57:29,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/a849eac66caa445cb7926ef2e07efcf1 is 175, key is test_row_0/A:col10/1733788648102/Put/seqid=0 2024-12-09T23:57:29,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741979_1155 (size=31105) 2024-12-09T23:57:29,020 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=152, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/a849eac66caa445cb7926ef2e07efcf1 2024-12-09T23:57:29,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/f0da2c21a16843ceadabb04044a80048 is 50, key is test_row_0/B:col10/1733788648102/Put/seqid=0 2024-12-09T23:57:29,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741980_1156 (size=12151) 2024-12-09T23:57:29,051 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/f0da2c21a16843ceadabb04044a80048 2024-12-09T23:57:29,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/d500431bd2574af897271c5ee43d73ec is 50, key is test_row_0/C:col10/1733788648102/Put/seqid=0 2024-12-09T23:57:29,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741981_1157 (size=12151) 2024-12-09T23:57:29,065 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/d500431bd2574af897271c5ee43d73ec 2024-12-09T23:57:29,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/a849eac66caa445cb7926ef2e07efcf1 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/a849eac66caa445cb7926ef2e07efcf1 2024-12-09T23:57:29,074 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/a849eac66caa445cb7926ef2e07efcf1, entries=150, sequenceid=152, filesize=30.4 K 2024-12-09T23:57:29,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/f0da2c21a16843ceadabb04044a80048 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/f0da2c21a16843ceadabb04044a80048 2024-12-09T23:57:29,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,080 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/f0da2c21a16843ceadabb04044a80048, entries=150, sequenceid=152, filesize=11.9 K 2024-12-09T23:57:29,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/d500431bd2574af897271c5ee43d73ec as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/d500431bd2574af897271c5ee43d73ec 2024-12-09T23:57:29,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,085 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/d500431bd2574af897271c5ee43d73ec, entries=150, sequenceid=152, filesize=11.9 K 2024-12-09T23:57:29,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,085 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for a71a18ca5bdcebe10547b57d8dd2e16f in 112ms, sequenceid=152, compaction requested=true 2024-12-09T23:57:29,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for a71a18ca5bdcebe10547b57d8dd2e16f: 2024-12-09T23:57:29,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:29,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-09T23:57:29,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-09T23:57:29,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,089 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-09T23:57:29,089 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 571 msec 2024-12-09T23:57:29,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,092 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 577 msec 2024-12-09T23:57:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T23:57:29,143 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:57:29,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,144 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:57:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-09T23:57:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T23:57:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,147 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:57:29,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,148 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:57:29,148 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:57:29,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T23:57:29,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T23:57:29,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:29,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,301 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing a71a18ca5bdcebe10547b57d8dd2e16f 3/3 column families, dataSize=6.71 KB heapSize=20.67 KB 2024-12-09T23:57:29,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=A 2024-12-09T23:57:29,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:29,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=B 2024-12-09T23:57:29,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:29,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=C 2024-12-09T23:57:29,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:29,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209e01bafe02617476aad804836631b8e84_a71a18ca5bdcebe10547b57d8dd2e16f is 50, key is test_row_1/A:col10/1733788649277/Put/seqid=0 2024-12-09T23:57:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:29,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:29,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741982_1158 (size=7324) 2024-12-09T23:57:29,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:29,362 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209e01bafe02617476aad804836631b8e84_a71a18ca5bdcebe10547b57d8dd2e16f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e01bafe02617476aad804836631b8e84_a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:29,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/c95153df518c4a86ae08056e24e8361e, store: [table=TestAcidGuarantees family=A region=a71a18ca5bdcebe10547b57d8dd2e16f] 2024-12-09T23:57:29,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/c95153df518c4a86ae08056e24e8361e is 175, key is test_row_1/A:col10/1733788649277/Put/seqid=0 2024-12-09T23:57:29,400 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733788649374, max timestamp=1733788649397Num uniques 859; 2024-12-09T23:57:29,400 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733788649374, max timestamp=1733788649397Num uniques 859; 2024-12-09T23:57:29,400 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733788649374, max timestamp=1733788649397Num uniques 859; 2024-12-09T23:57:29,400 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T23:57:29,400 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T23:57:29,400 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T23:57:29,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741983_1159 (size=13815) 2024-12-09T23:57:29,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T23:57:29,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733788649443, max timestamp=1733788649470Num uniques 1068; 2024-12-09T23:57:29,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-12-09T23:57:29,473 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733788649443, max timestamp=1733788649470Num uniques 1068; 2024-12-09T23:57:29,473 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-12-09T23:57:29,473 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733788649443, max timestamp=1733788649470Num uniques 1068; 2024-12-09T23:57:29,473 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-12-09T23:57:29,521 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733788649492, max timestamp=1733788649518Num uniques 1127; 2024-12-09T23:57:29,521 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-12-09T23:57:29,521 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733788649492, max timestamp=1733788649518Num uniques 1127; 2024-12-09T23:57:29,521 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-12-09T23:57:29,522 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733788649492, max timestamp=1733788649518Num uniques 1127; 2024-12-09T23:57:29,522 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-12-09T23:57:29,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733788649544, max timestamp=1733788649574Num uniques 1086; 2024-12-09T23:57:29,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-09T23:57:29,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733788649544, max timestamp=1733788649574Num uniques 1086; 2024-12-09T23:57:29,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-09T23:57:29,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733788649544, max timestamp=1733788649574Num uniques 1086; 2024-12-09T23:57:29,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-09T23:57:29,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733788649606, max timestamp=1733788649626Num uniques 1095; 2024-12-09T23:57:29,629 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733788649606, max timestamp=1733788649626Num uniques 1095; 2024-12-09T23:57:29,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-12-09T23:57:29,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-12-09T23:57:29,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733788649617, max timestamp=1733788649626Num uniques 1095; 2024-12-09T23:57:29,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-12-09T23:57:29,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733788649670, max timestamp=1733788649697Num uniques 1104; 2024-12-09T23:57:29,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733788649670, max timestamp=1733788649697Num uniques 1104; 2024-12-09T23:57:29,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-12-09T23:57:29,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-12-09T23:57:29,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733788649670, max timestamp=1733788649697Num uniques 1054; 2024-12-09T23:57:29,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-12-09T23:57:29,765 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733788649740, max timestamp=1733788649762Num uniques 1063; 2024-12-09T23:57:29,765 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733788649740, max timestamp=1733788649762Num uniques 1063; 2024-12-09T23:57:29,765 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733788649740, max timestamp=1733788649762Num uniques 1063; 2024-12-09T23:57:29,765 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-12-09T23:57:29,765 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-12-09T23:57:29,765 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-12-09T23:57:29,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T23:57:29,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733788649803, max timestamp=1733788649831Num uniques 1072; 2024-12-09T23:57:29,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-12-09T23:57:29,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733788649803, max timestamp=1733788649831Num uniques 1072; 2024-12-09T23:57:29,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-12-09T23:57:29,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733788649803, max timestamp=1733788649831Num uniques 1072; 2024-12-09T23:57:29,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-12-09T23:57:29,858 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=2.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/c95153df518c4a86ae08056e24e8361e 2024-12-09T23:57:29,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/af2060c672074ab99f905e7f60dd5f2f is 50, key is test_row_1/B:col10/1733788649277/Put/seqid=0 2024-12-09T23:57:29,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741984_1160 (size=7365) 2024-12-09T23:57:29,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733788649871, max timestamp=1733788649890Num uniques 1031; 2024-12-09T23:57:29,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-12-09T23:57:29,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733788649871, max timestamp=1733788649890Num uniques 1031; 2024-12-09T23:57:29,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-12-09T23:57:29,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733788649871, max timestamp=1733788649890Num uniques 1031; 2024-12-09T23:57:29,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-12-09T23:57:29,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733788649890, max timestamp=1733788649953Num uniques 1040; 2024-12-09T23:57:29,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733788649890, max timestamp=1733788649953Num uniques 1040; 2024-12-09T23:57:29,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-12-09T23:57:29,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-12-09T23:57:29,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733788649890, max timestamp=1733788649953Num uniques 1040; 2024-12-09T23:57:29,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-12-09T23:57:30,037 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1733788650001, max timestamp=1733788650034Num uniques 1099; 2024-12-09T23:57:30,037 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1733788650001, max timestamp=1733788650034Num uniques 1099; 2024-12-09T23:57:30,037 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=9114, suffix heap size=142264,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=199 2024-12-09T23:57:30,037 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=9114, suffix heap size=142264,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=199 2024-12-09T23:57:30,037 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1733788650001, max timestamp=1733788650034Num uniques 1099; 2024-12-09T23:57:30,037 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=9114, suffix heap size=142264,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=199 2024-12-09T23:57:30,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1733788650072, max timestamp=1733788650103Num uniques 1058; 2024-12-09T23:57:30,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1733788650072, max timestamp=1733788650103Num uniques 1058; 2024-12-09T23:57:30,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1733788650072, max timestamp=1733788650103Num uniques 1058; 2024-12-09T23:57:30,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-12-09T23:57:30,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-12-09T23:57:30,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-12-09T23:57:30,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1733788650155, max timestamp=1733788650171Num uniques 1017; 2024-12-09T23:57:30,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=9932, suffix heap size=133736,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=217 2024-12-09T23:57:30,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1733788650155, max timestamp=1733788650171Num uniques 1017; 2024-12-09T23:57:30,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=9932, suffix heap size=133736,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=217 2024-12-09T23:57:30,176 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1733788650155, max timestamp=1733788650171Num uniques 1017; 2024-12-09T23:57:30,176 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=9932, suffix heap size=133736,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=217 2024-12-09T23:57:30,240 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1733788650204, max timestamp=1733788650237Num uniques 1076; 2024-12-09T23:57:30,240 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=10346, suffix heap size=139872,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=226 2024-12-09T23:57:30,240 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1733788650204, max timestamp=1733788650237Num uniques 1076; 2024-12-09T23:57:30,240 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=10346, suffix heap size=139872,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=226 2024-12-09T23:57:30,240 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1733788650204, max timestamp=1733788650237Num uniques 1076; 2024-12-09T23:57:30,240 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=10346, suffix heap size=139872,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=226 2024-12-09T23:57:30,271 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/af2060c672074ab99f905e7f60dd5f2f 2024-12-09T23:57:30,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/b7006eca5c4b4d5384faa8f7ad8dc6c8 is 50, key is test_row_1/C:col10/1733788649277/Put/seqid=0 2024-12-09T23:57:30,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741985_1161 (size=7365) 2024-12-09T23:57:30,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T23:57:30,297 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=285, cellSize=13050, totalHeapSize=29880, min timestamp=1733788650280, max timestamp=1733788650294Num uniques 1085; 2024-12-09T23:57:30,297 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=285, cellSize=13050, totalHeapSize=29880, min timestamp=1733788650280, max timestamp=1733788650294Num uniques 1085; 2024-12-09T23:57:30,297 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=285, cellSize=13050, totalHeapSize=29880, min timestamp=1733788650280, max timestamp=1733788650294Num uniques 1085; 2024-12-09T23:57:30,297 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=13050, suffix heap size=140808,new segment heap size=29880 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=285 2024-12-09T23:57:30,297 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=13050, suffix heap size=140808,new segment heap size=29880 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=285 2024-12-09T23:57:30,297 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=13050, suffix heap size=140808,new segment heap size=29880 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=285 2024-12-09T23:57:30,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8884, totalHeapSize=20416, min timestamp=1733788650320, max timestamp=1733788650380Num uniques 1144; 2024-12-09T23:57:30,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8884, totalHeapSize=20416, min timestamp=1733788650320, max timestamp=1733788650380Num uniques 1144; 2024-12-09T23:57:30,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8884, totalHeapSize=20416, min timestamp=1733788650320, max timestamp=1733788650380Num uniques 1144; 2024-12-09T23:57:30,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52394, new segment data size=8884, suffix heap size=146944,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1144, new segment cells count=194 2024-12-09T23:57:30,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52394, new segment data size=8884, suffix heap size=146944,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1144, new segment cells count=194 2024-12-09T23:57:30,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52394, new segment data size=8884, suffix heap size=146944,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1144, new segment cells count=194 2024-12-09T23:57:30,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1733788650428, max timestamp=1733788650447Num uniques 1053; 2024-12-09T23:57:30,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1733788650428, max timestamp=1733788650447Num uniques 1053; 2024-12-09T23:57:30,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=7005, suffix heap size=137480,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=153 2024-12-09T23:57:30,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=7005, suffix heap size=137480,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=153 2024-12-09T23:57:30,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1733788650428, max timestamp=1733788650447Num uniques 1053; 2024-12-09T23:57:30,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=7005, suffix heap size=137480,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=153 2024-12-09T23:57:30,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=212, cellSize=9702, totalHeapSize=22288, min timestamp=1733788650489, max timestamp=1733788650509Num uniques 1012; 2024-12-09T23:57:30,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=212, cellSize=9702, totalHeapSize=22288, min timestamp=1733788650489, max timestamp=1733788650509Num uniques 1012; 2024-12-09T23:57:30,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=212, cellSize=9702, totalHeapSize=22288, min timestamp=1733788650489, max timestamp=1733788650509Num uniques 1012; 2024-12-09T23:57:30,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46342, new segment data size=9702, suffix heap size=133216,new segment heap size=22288 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=212 2024-12-09T23:57:30,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46342, new segment data size=9702, suffix heap size=133216,new segment heap size=22288 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=212 2024-12-09T23:57:30,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46342, new segment data size=9702, suffix heap size=133216,new segment heap size=22288 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=212 2024-12-09T23:57:30,586 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=171, cellSize=7826, totalHeapSize=18024, min timestamp=1733788650558, max timestamp=1733788650583Num uniques 1071; 2024-12-09T23:57:30,586 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49046, new segment data size=7826, suffix heap size=139352,new segment heap size=18024 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1071, new segment cells count=171 2024-12-09T23:57:30,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=171, cellSize=7826, totalHeapSize=18024, min timestamp=1733788650558, max timestamp=1733788650583Num uniques 1071; 2024-12-09T23:57:30,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49046, new segment data size=7826, suffix heap size=139352,new segment heap size=18024 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1071, new segment cells count=171 2024-12-09T23:57:30,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=171, cellSize=7826, totalHeapSize=18024, min timestamp=1733788650558, max timestamp=1733788650583Num uniques 1071; 2024-12-09T23:57:30,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49046, new segment data size=7826, suffix heap size=139352,new segment heap size=18024 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1071, new segment cells count=171 2024-12-09T23:57:30,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=280, cellSize=12820, totalHeapSize=29360, min timestamp=1733788650596, max timestamp=1733788650649Num uniques 1030; 2024-12-09T23:57:30,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47170, new segment data size=12820, suffix heap size=135088,new segment heap size=29360 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1030, new segment cells count=280 2024-12-09T23:57:30,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=180, cellSize=8240, totalHeapSize=18960, min timestamp=1733788650621, max timestamp=1733788650649Num uniques 1030; 2024-12-09T23:57:30,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47170, new segment data size=8240, suffix heap size=135088,new segment heap size=18960 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1030, new segment cells count=180 2024-12-09T23:57:30,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733788650621, max timestamp=1733788650649Num uniques 1030; 2024-12-09T23:57:30,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47170, new segment data size=6870, suffix heap size=135088,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1030, new segment cells count=150 2024-12-09T23:57:30,682 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/b7006eca5c4b4d5384faa8f7ad8dc6c8 2024-12-09T23:57:30,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/c95153df518c4a86ae08056e24e8361e as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/c95153df518c4a86ae08056e24e8361e 2024-12-09T23:57:30,694 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/c95153df518c4a86ae08056e24e8361e, entries=50, sequenceid=156, filesize=13.5 K 2024-12-09T23:57:30,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/af2060c672074ab99f905e7f60dd5f2f as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/af2060c672074ab99f905e7f60dd5f2f 2024-12-09T23:57:30,703 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/af2060c672074ab99f905e7f60dd5f2f, entries=50, sequenceid=156, filesize=7.2 K 2024-12-09T23:57:30,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/b7006eca5c4b4d5384faa8f7ad8dc6c8 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/b7006eca5c4b4d5384faa8f7ad8dc6c8 2024-12-09T23:57:30,709 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/b7006eca5c4b4d5384faa8f7ad8dc6c8, entries=50, sequenceid=156, filesize=7.2 K 2024-12-09T23:57:30,710 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=110.48 KB/113130 for a71a18ca5bdcebe10547b57d8dd2e16f in 1408ms, sequenceid=156, compaction requested=true 2024-12-09T23:57:30,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for a71a18ca5bdcebe10547b57d8dd2e16f: 2024-12-09T23:57:30,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:30,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-09T23:57:30,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-09T23:57:30,713 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-09T23:57:30,713 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5640 sec 2024-12-09T23:57:30,715 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.5700 sec 2024-12-09T23:57:30,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:30,715 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a71a18ca5bdcebe10547b57d8dd2e16f 3/3 column families, dataSize=117.19 KB heapSize=330.36 KB 2024-12-09T23:57:30,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=A 2024-12-09T23:57:30,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:57:30,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=B 2024-12-09T23:57:30,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:57:30,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=C 2024-12-09T23:57:30,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:57:30,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209a090ea079782460e9f9430bf86de32ea_a71a18ca5bdcebe10547b57d8dd2e16f is 50, key is test_row_0/A:col10/1733788650711/Put/seqid=0 2024-12-09T23:57:30,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741986_1162 (size=14994) 2024-12-09T23:57:30,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:30,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 451 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788710775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:30,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:30,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:30,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:30,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:30,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 452 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788710893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:30,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:30,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:30,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:31,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 453 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788711102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:31,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:31,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:31,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:31,133 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:31,137 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209a090ea079782460e9f9430bf86de32ea_a71a18ca5bdcebe10547b57d8dd2e16f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a090ea079782460e9f9430bf86de32ea_a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:31,138 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/4650fa800fb849599c6bdaac77d92b1d, store: [table=TestAcidGuarantees family=A region=a71a18ca5bdcebe10547b57d8dd2e16f] 2024-12-09T23:57:31,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/4650fa800fb849599c6bdaac77d92b1d is 175, key is test_row_0/A:col10/1733788650711/Put/seqid=0 2024-12-09T23:57:31,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741987_1163 (size=39949) 2024-12-09T23:57:31,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T23:57:31,293 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:57:31,294 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:57:31,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-09T23:57:31,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T23:57:31,295 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:57:31,296 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:57:31,296 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:57:31,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:31,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1733788711293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:31,300 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_2' in a71a18ca5bdcebe10547b57d8dd2e16f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:57:31,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:31,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:31,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:31,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:31,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42764 deadline: 1733788711312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:31,317 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_0' in a71a18ca5bdcebe10547b57d8dd2e16f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:57:31,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:31,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:31,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:31,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:31,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42836 deadline: 1733788711323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:31,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:31,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42810 deadline: 1733788711323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:31,328 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_2' in a71a18ca5bdcebe10547b57d8dd2e16f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:57:31,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:31,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:31,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:31,329 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_1' in a71a18ca5bdcebe10547b57d8dd2e16f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:57:31,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:31,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:31,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T23:57:31,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 454 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788711413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:31,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:31,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:31,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:31,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T23:57:31,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:31,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:31,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:31,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:31,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:31,544 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=516, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/4650fa800fb849599c6bdaac77d92b1d 2024-12-09T23:57:31,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/b0e693f4e46c47fcabc78df326a8dac7 is 50, key is test_row_0/B:col10/1733788650711/Put/seqid=0 2024-12-09T23:57:31,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741988_1164 (size=12301) 2024-12-09T23:57:31,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T23:57:31,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:31,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:31,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:31,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:31,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:31,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T23:57:31,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T23:57:31,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:31,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:31,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:31,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:31,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:31,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:31,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T23:57:31,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:31,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:31,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:31,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:31,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:31,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:31,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T23:57:31,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:31,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 455 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788711922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:31,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:31,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:31,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:31,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/b0e693f4e46c47fcabc78df326a8dac7 2024-12-09T23:57:31,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/4db13686f5f14bff9874c3c358508621 is 50, key is test_row_0/C:col10/1733788650711/Put/seqid=0 2024-12-09T23:57:31,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741989_1165 (size=12301) 2024-12-09T23:57:32,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T23:57:32,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:32,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:32,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:32,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:32,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:32,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:32,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T23:57:32,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:32,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:32,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:32,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:32,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:32,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:32,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T23:57:32,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:32,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:32,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:32,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:32,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:32,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:32,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=36.68 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/4db13686f5f14bff9874c3c358508621 2024-12-09T23:57:32,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/4650fa800fb849599c6bdaac77d92b1d as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/4650fa800fb849599c6bdaac77d92b1d 2024-12-09T23:57:32,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/4650fa800fb849599c6bdaac77d92b1d, entries=200, sequenceid=516, filesize=39.0 K 2024-12-09T23:57:32,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/b0e693f4e46c47fcabc78df326a8dac7 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/b0e693f4e46c47fcabc78df326a8dac7 2024-12-09T23:57:32,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/b0e693f4e46c47fcabc78df326a8dac7, entries=150, sequenceid=516, filesize=12.0 K 2024-12-09T23:57:32,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/4db13686f5f14bff9874c3c358508621 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/4db13686f5f14bff9874c3c358508621 2024-12-09T23:57:32,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/4db13686f5f14bff9874c3c358508621, entries=150, sequenceid=516, filesize=12.0 K 2024-12-09T23:57:32,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~117.19 KB/120000, heapSize ~330.31 KB/338240, currentSize=67.09 KB/68700 for a71a18ca5bdcebe10547b57d8dd2e16f in 1673ms, sequenceid=516, compaction requested=true 2024-12-09T23:57:32,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a71a18ca5bdcebe10547b57d8dd2e16f: 2024-12-09T23:57:32,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71a18ca5bdcebe10547b57d8dd2e16f:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:57:32,388 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-09T23:57:32,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:32,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71a18ca5bdcebe10547b57d8dd2e16f:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:57:32,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:32,389 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-09T23:57:32,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71a18ca5bdcebe10547b57d8dd2e16f:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:57:32,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:57:32,390 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 56109 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-09T23:57:32,390 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 155663 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-09T23:57:32,390 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): a71a18ca5bdcebe10547b57d8dd2e16f/A is initiating minor compaction (all files) 2024-12-09T23:57:32,390 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): a71a18ca5bdcebe10547b57d8dd2e16f/B is initiating minor compaction (all files) 2024-12-09T23:57:32,390 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71a18ca5bdcebe10547b57d8dd2e16f/A in TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:32,390 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71a18ca5bdcebe10547b57d8dd2e16f/B in TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:32,390 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/d25b341c859547b9bb52e351f1a69f09, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/92dc398ac9ba4bbb93cb2d4b9538ff53, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/a849eac66caa445cb7926ef2e07efcf1, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/c95153df518c4a86ae08056e24e8361e, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/4650fa800fb849599c6bdaac77d92b1d] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp, totalSize=152.0 K 2024-12-09T23:57:32,390 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/b11fe4f3cd78426f82d647655e9cabf8, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/3b17060f2e0942238e17b8a5bcb382df, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/f0da2c21a16843ceadabb04044a80048, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/af2060c672074ab99f905e7f60dd5f2f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/b0e693f4e46c47fcabc78df326a8dac7] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp, totalSize=54.8 K 2024-12-09T23:57:32,390 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:32,390 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. files: [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/d25b341c859547b9bb52e351f1a69f09, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/92dc398ac9ba4bbb93cb2d4b9538ff53, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/a849eac66caa445cb7926ef2e07efcf1, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/c95153df518c4a86ae08056e24e8361e, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/4650fa800fb849599c6bdaac77d92b1d] 2024-12-09T23:57:32,391 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting d25b341c859547b9bb52e351f1a69f09, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733788646729 2024-12-09T23:57:32,391 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting b11fe4f3cd78426f82d647655e9cabf8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733788646729 2024-12-09T23:57:32,392 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92dc398ac9ba4bbb93cb2d4b9538ff53, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733788646876 2024-12-09T23:57:32,392 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b17060f2e0942238e17b8a5bcb382df, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733788646876 2024-12-09T23:57:32,392 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting a849eac66caa445cb7926ef2e07efcf1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733788648092 2024-12-09T23:57:32,392 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting f0da2c21a16843ceadabb04044a80048, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733788648092 2024-12-09T23:57:32,392 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting c95153df518c4a86ae08056e24e8361e, keycount=50, bloomtype=ROW, size=13.5 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733788649277 2024-12-09T23:57:32,392 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting af2060c672074ab99f905e7f60dd5f2f, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733788649277 2024-12-09T23:57:32,392 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4650fa800fb849599c6bdaac77d92b1d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=516, earliestPutTs=1733788650681 2024-12-09T23:57:32,392 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting b0e693f4e46c47fcabc78df326a8dac7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=516, earliestPutTs=1733788650693 2024-12-09T23:57:32,415 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a71a18ca5bdcebe10547b57d8dd2e16f] 2024-12-09T23:57:32,418 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120937e1295b5e4f410d8336731c76fe9dcf_a71a18ca5bdcebe10547b57d8dd2e16f store=[table=TestAcidGuarantees family=A region=a71a18ca5bdcebe10547b57d8dd2e16f] 2024-12-09T23:57:32,418 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71a18ca5bdcebe10547b57d8dd2e16f#B#compaction#138 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:57:32,418 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/ba1bd1c900eb49748019ee06fe776c8c is 50, key is test_row_0/B:col10/1733788650711/Put/seqid=0 2024-12-09T23:57:32,420 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120937e1295b5e4f410d8336731c76fe9dcf_a71a18ca5bdcebe10547b57d8dd2e16f, store=[table=TestAcidGuarantees family=A region=a71a18ca5bdcebe10547b57d8dd2e16f] 2024-12-09T23:57:32,421 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120937e1295b5e4f410d8336731c76fe9dcf_a71a18ca5bdcebe10547b57d8dd2e16f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a71a18ca5bdcebe10547b57d8dd2e16f] 2024-12-09T23:57:32,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741990_1166 (size=12711) 2024-12-09T23:57:32,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741991_1167 (size=4469) 2024-12-09T23:57:32,432 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71a18ca5bdcebe10547b57d8dd2e16f#A#compaction#139 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:57:32,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T23:57:32,433 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/bc66703e1b7249b39605fb65f922abe1 is 175, key is test_row_0/A:col10/1733788650711/Put/seqid=0 2024-12-09T23:57:32,436 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/ba1bd1c900eb49748019ee06fe776c8c as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/ba1bd1c900eb49748019ee06fe776c8c 2024-12-09T23:57:32,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741992_1168 (size=31665) 2024-12-09T23:57:32,442 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/bc66703e1b7249b39605fb65f922abe1 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/bc66703e1b7249b39605fb65f922abe1 2024-12-09T23:57:32,442 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in a71a18ca5bdcebe10547b57d8dd2e16f/B of a71a18ca5bdcebe10547b57d8dd2e16f into ba1bd1c900eb49748019ee06fe776c8c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:32,442 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71a18ca5bdcebe10547b57d8dd2e16f: 2024-12-09T23:57:32,442 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., storeName=a71a18ca5bdcebe10547b57d8dd2e16f/B, priority=11, startTime=1733788652389; duration=0sec 2024-12-09T23:57:32,443 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:57:32,443 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71a18ca5bdcebe10547b57d8dd2e16f:B 2024-12-09T23:57:32,443 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-09T23:57:32,445 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 56109 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-09T23:57:32,445 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): a71a18ca5bdcebe10547b57d8dd2e16f/C is initiating minor compaction (all files) 2024-12-09T23:57:32,445 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71a18ca5bdcebe10547b57d8dd2e16f/C in TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:32,445 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/fcc07f32e5f24b33b8cb5a832b31846e, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/a27a61b5e9f7463c9e7999a53dd35a83, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/d500431bd2574af897271c5ee43d73ec, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/b7006eca5c4b4d5384faa8f7ad8dc6c8, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/4db13686f5f14bff9874c3c358508621] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp, totalSize=54.8 K 2024-12-09T23:57:32,445 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting fcc07f32e5f24b33b8cb5a832b31846e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733788646729 2024-12-09T23:57:32,446 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting a27a61b5e9f7463c9e7999a53dd35a83, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733788646876 2024-12-09T23:57:32,446 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting d500431bd2574af897271c5ee43d73ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733788648092 2024-12-09T23:57:32,446 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting b7006eca5c4b4d5384faa8f7ad8dc6c8, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733788649277 2024-12-09T23:57:32,446 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 4db13686f5f14bff9874c3c358508621, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=516, earliestPutTs=1733788650693 2024-12-09T23:57:32,448 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in a71a18ca5bdcebe10547b57d8dd2e16f/A of a71a18ca5bdcebe10547b57d8dd2e16f into bc66703e1b7249b39605fb65f922abe1(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:32,448 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71a18ca5bdcebe10547b57d8dd2e16f: 2024-12-09T23:57:32,448 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., storeName=a71a18ca5bdcebe10547b57d8dd2e16f/A, priority=11, startTime=1733788652388; duration=0sec 2024-12-09T23:57:32,448 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:32,448 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71a18ca5bdcebe10547b57d8dd2e16f:A 2024-12-09T23:57:32,456 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71a18ca5bdcebe10547b57d8dd2e16f#C#compaction#140 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:57:32,456 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/5245319cc8e842c2ad6556172f106b3e is 50, key is test_row_0/C:col10/1733788650711/Put/seqid=0 2024-12-09T23:57:32,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741993_1169 (size=12711) 2024-12-09T23:57:32,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T23:57:32,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:32,516 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing a71a18ca5bdcebe10547b57d8dd2e16f 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T23:57:32,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=A 2024-12-09T23:57:32,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:32,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=B 2024-12-09T23:57:32,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:32,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=C 2024-12-09T23:57:32,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:32,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412090a3a72e2166040a9b42e7e5dfc690564_a71a18ca5bdcebe10547b57d8dd2e16f is 50, key is test_row_0/A:col10/1733788650750/Put/seqid=0 2024-12-09T23:57:32,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741994_1170 (size=12454) 2024-12-09T23:57:32,864 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/5245319cc8e842c2ad6556172f106b3e as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/5245319cc8e842c2ad6556172f106b3e 2024-12-09T23:57:32,869 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in a71a18ca5bdcebe10547b57d8dd2e16f/C of a71a18ca5bdcebe10547b57d8dd2e16f into 5245319cc8e842c2ad6556172f106b3e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:32,869 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71a18ca5bdcebe10547b57d8dd2e16f: 2024-12-09T23:57:32,870 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., storeName=a71a18ca5bdcebe10547b57d8dd2e16f/C, priority=11, startTime=1733788652389; duration=0sec 2024-12-09T23:57:32,870 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:32,870 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71a18ca5bdcebe10547b57d8dd2e16f:C 2024-12-09T23:57:32,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:32,930 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412090a3a72e2166040a9b42e7e5dfc690564_a71a18ca5bdcebe10547b57d8dd2e16f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412090a3a72e2166040a9b42e7e5dfc690564_a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:32,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/3e20fa792dc3420db4ee60916ea43146, store: [table=TestAcidGuarantees family=A region=a71a18ca5bdcebe10547b57d8dd2e16f] 2024-12-09T23:57:32,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/3e20fa792dc3420db4ee60916ea43146 is 175, key is test_row_0/A:col10/1733788650750/Put/seqid=0 2024-12-09T23:57:32,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741995_1171 (size=31255) 2024-12-09T23:57:32,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:32,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:32,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:32,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 472 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788712982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:32,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:32,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:32,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:33,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:33,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 473 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788713093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:33,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:33,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:33,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:33,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:33,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 474 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788713302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:33,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:33,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:33,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:33,336 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=531, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/3e20fa792dc3420db4ee60916ea43146 2024-12-09T23:57:33,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/a5393b3abd0b4c619c05d06f15a66044 is 50, key is test_row_0/B:col10/1733788650750/Put/seqid=0 2024-12-09T23:57:33,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741996_1172 (size=12301) 2024-12-09T23:57:33,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T23:57:33,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:33,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 475 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788713613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:33,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:33,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:33,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:33,747 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/a5393b3abd0b4c619c05d06f15a66044 2024-12-09T23:57:33,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/f50fbcc7323b4c76b740edd2e8a52a14 is 50, key is test_row_0/C:col10/1733788650750/Put/seqid=0 2024-12-09T23:57:33,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741997_1173 (size=12301) 2024-12-09T23:57:34,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:34,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 476 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788714122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:34,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:34,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:34,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:34,158 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/f50fbcc7323b4c76b740edd2e8a52a14 2024-12-09T23:57:34,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/3e20fa792dc3420db4ee60916ea43146 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/3e20fa792dc3420db4ee60916ea43146 2024-12-09T23:57:34,165 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/3e20fa792dc3420db4ee60916ea43146, entries=150, sequenceid=531, filesize=30.5 K 2024-12-09T23:57:34,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/a5393b3abd0b4c619c05d06f15a66044 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/a5393b3abd0b4c619c05d06f15a66044 2024-12-09T23:57:34,170 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/a5393b3abd0b4c619c05d06f15a66044, entries=150, sequenceid=531, filesize=12.0 K 2024-12-09T23:57:34,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/f50fbcc7323b4c76b740edd2e8a52a14 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/f50fbcc7323b4c76b740edd2e8a52a14 2024-12-09T23:57:34,176 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/f50fbcc7323b4c76b740edd2e8a52a14, entries=150, sequenceid=531, filesize=12.0 K 2024-12-09T23:57:34,177 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for a71a18ca5bdcebe10547b57d8dd2e16f in 1661ms, sequenceid=531, compaction requested=false 2024-12-09T23:57:34,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for a71a18ca5bdcebe10547b57d8dd2e16f: 2024-12-09T23:57:34,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:34,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-09T23:57:34,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-09T23:57:34,179 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-09T23:57:34,179 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8820 sec 2024-12-09T23:57:34,181 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.8860 sec 2024-12-09T23:57:35,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:35,134 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a71a18ca5bdcebe10547b57d8dd2e16f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T23:57:35,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=A 2024-12-09T23:57:35,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:35,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=B 2024-12-09T23:57:35,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:35,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=C 2024-12-09T23:57:35,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:35,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ebf9aa6026994afbafa1d02cf925307d_a71a18ca5bdcebe10547b57d8dd2e16f is 50, key is test_row_0/A:col10/1733788655132/Put/seqid=0 2024-12-09T23:57:35,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741998_1174 (size=14994) 2024-12-09T23:57:35,149 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:35,154 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ebf9aa6026994afbafa1d02cf925307d_a71a18ca5bdcebe10547b57d8dd2e16f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ebf9aa6026994afbafa1d02cf925307d_a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:35,155 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/fb772c8b7fe24e1f83e1f82c88211d48, store: [table=TestAcidGuarantees family=A region=a71a18ca5bdcebe10547b57d8dd2e16f] 2024-12-09T23:57:35,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/fb772c8b7fe24e1f83e1f82c88211d48 is 175, key is test_row_0/A:col10/1733788655132/Put/seqid=0 2024-12-09T23:57:35,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741999_1175 (size=39949) 2024-12-09T23:57:35,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:35,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 487 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788715163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:35,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:35,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:35,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:35,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:35,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 488 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788715272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:35,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:35,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:35,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:35,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T23:57:35,463 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:57:35,463 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:57:35,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-09T23:57:35,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T23:57:35,465 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:57:35,465 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:57:35,466 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:57:35,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:35,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 489 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788715482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:35,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:35,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:35,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:35,561 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=552, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/fb772c8b7fe24e1f83e1f82c88211d48 2024-12-09T23:57:35,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/59580fa685b5421ea585c288af558d1e is 50, key is test_row_0/B:col10/1733788655132/Put/seqid=0 2024-12-09T23:57:35,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742000_1176 (size=12301) 2024-12-09T23:57:35,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T23:57:35,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T23:57:35,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:35,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:35,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:35,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:35,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:35,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:35,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T23:57:35,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:35,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:35,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:35,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:35,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:35,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T23:57:35,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 490 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788715793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:35,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:35,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:35,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:35,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T23:57:35,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:35,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:35,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:35,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:35,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:35,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=552 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/59580fa685b5421ea585c288af558d1e 2024-12-09T23:57:35,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/d415a526ba3746289f01150529418575 is 50, key is test_row_0/C:col10/1733788655132/Put/seqid=0 2024-12-09T23:57:35,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742001_1177 (size=12301) 2024-12-09T23:57:36,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T23:57:36,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:36,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:36,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:36,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:36,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:36,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:36,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T23:57:36,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T23:57:36,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:36,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:36,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:36,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:36,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:36,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:36,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:36,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 491 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788716303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:36,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:36,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:36,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:36,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T23:57:36,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:36,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:36,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:36,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:36,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:36,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=552 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/d415a526ba3746289f01150529418575 2024-12-09T23:57:36,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:36,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/fb772c8b7fe24e1f83e1f82c88211d48 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/fb772c8b7fe24e1f83e1f82c88211d48 2024-12-09T23:57:36,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/fb772c8b7fe24e1f83e1f82c88211d48, entries=200, sequenceid=552, filesize=39.0 K 2024-12-09T23:57:36,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/59580fa685b5421ea585c288af558d1e as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/59580fa685b5421ea585c288af558d1e 2024-12-09T23:57:36,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/59580fa685b5421ea585c288af558d1e, entries=150, sequenceid=552, filesize=12.0 K 2024-12-09T23:57:36,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/d415a526ba3746289f01150529418575 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/d415a526ba3746289f01150529418575 2024-12-09T23:57:36,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/d415a526ba3746289f01150529418575, entries=150, sequenceid=552, filesize=12.0 K 2024-12-09T23:57:36,404 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a71a18ca5bdcebe10547b57d8dd2e16f in 1270ms, sequenceid=552, compaction requested=true 2024-12-09T23:57:36,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a71a18ca5bdcebe10547b57d8dd2e16f: 2024-12-09T23:57:36,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71a18ca5bdcebe10547b57d8dd2e16f:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:57:36,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:36,404 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:57:36,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71a18ca5bdcebe10547b57d8dd2e16f:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:57:36,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:36,404 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:57:36,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71a18ca5bdcebe10547b57d8dd2e16f:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:57:36,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:57:36,406 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102869 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:57:36,406 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:57:36,406 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): a71a18ca5bdcebe10547b57d8dd2e16f/A is initiating minor compaction (all files) 2024-12-09T23:57:36,406 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): a71a18ca5bdcebe10547b57d8dd2e16f/B is initiating minor compaction (all files) 2024-12-09T23:57:36,406 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71a18ca5bdcebe10547b57d8dd2e16f/A in TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:36,406 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71a18ca5bdcebe10547b57d8dd2e16f/B in TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:36,406 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/bc66703e1b7249b39605fb65f922abe1, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/3e20fa792dc3420db4ee60916ea43146, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/fb772c8b7fe24e1f83e1f82c88211d48] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp, totalSize=100.5 K 2024-12-09T23:57:36,406 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/ba1bd1c900eb49748019ee06fe776c8c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/a5393b3abd0b4c619c05d06f15a66044, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/59580fa685b5421ea585c288af558d1e] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp, totalSize=36.4 K 2024-12-09T23:57:36,406 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:36,406 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. files: [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/bc66703e1b7249b39605fb65f922abe1, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/3e20fa792dc3420db4ee60916ea43146, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/fb772c8b7fe24e1f83e1f82c88211d48] 2024-12-09T23:57:36,406 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting ba1bd1c900eb49748019ee06fe776c8c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=516, earliestPutTs=1733788650693 2024-12-09T23:57:36,406 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc66703e1b7249b39605fb65f922abe1, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=516, earliestPutTs=1733788650693 2024-12-09T23:57:36,407 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting a5393b3abd0b4c619c05d06f15a66044, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1733788650750 2024-12-09T23:57:36,407 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e20fa792dc3420db4ee60916ea43146, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1733788650750 2024-12-09T23:57:36,407 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 59580fa685b5421ea585c288af558d1e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=552, earliestPutTs=1733788652975 2024-12-09T23:57:36,407 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb772c8b7fe24e1f83e1f82c88211d48, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=552, earliestPutTs=1733788652971 2024-12-09T23:57:36,415 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a71a18ca5bdcebe10547b57d8dd2e16f] 2024-12-09T23:57:36,417 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71a18ca5bdcebe10547b57d8dd2e16f#B#compaction#147 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:57:36,417 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/64adf21baa2b4395a6ffdda47cd491bf is 50, key is test_row_0/B:col10/1733788655132/Put/seqid=0 2024-12-09T23:57:36,427 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209aaa2952655fc46bd834e7e088d484b1f_a71a18ca5bdcebe10547b57d8dd2e16f store=[table=TestAcidGuarantees family=A region=a71a18ca5bdcebe10547b57d8dd2e16f] 2024-12-09T23:57:36,429 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209aaa2952655fc46bd834e7e088d484b1f_a71a18ca5bdcebe10547b57d8dd2e16f, store=[table=TestAcidGuarantees family=A region=a71a18ca5bdcebe10547b57d8dd2e16f] 2024-12-09T23:57:36,429 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209aaa2952655fc46bd834e7e088d484b1f_a71a18ca5bdcebe10547b57d8dd2e16f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a71a18ca5bdcebe10547b57d8dd2e16f] 2024-12-09T23:57:36,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742002_1178 (size=12813) 2024-12-09T23:57:36,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742003_1179 (size=4469) 2024-12-09T23:57:36,442 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/64adf21baa2b4395a6ffdda47cd491bf as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/64adf21baa2b4395a6ffdda47cd491bf 2024-12-09T23:57:36,447 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71a18ca5bdcebe10547b57d8dd2e16f/B of a71a18ca5bdcebe10547b57d8dd2e16f into 64adf21baa2b4395a6ffdda47cd491bf(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:36,447 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71a18ca5bdcebe10547b57d8dd2e16f: 2024-12-09T23:57:36,447 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., storeName=a71a18ca5bdcebe10547b57d8dd2e16f/B, priority=13, startTime=1733788656404; duration=0sec 2024-12-09T23:57:36,447 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:57:36,447 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71a18ca5bdcebe10547b57d8dd2e16f:B 2024-12-09T23:57:36,447 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:57:36,448 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:57:36,448 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): a71a18ca5bdcebe10547b57d8dd2e16f/C is initiating minor compaction (all files) 2024-12-09T23:57:36,448 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71a18ca5bdcebe10547b57d8dd2e16f/C in TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:36,448 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/5245319cc8e842c2ad6556172f106b3e, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/f50fbcc7323b4c76b740edd2e8a52a14, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/d415a526ba3746289f01150529418575] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp, totalSize=36.4 K 2024-12-09T23:57:36,448 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 5245319cc8e842c2ad6556172f106b3e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=516, earliestPutTs=1733788650693 2024-12-09T23:57:36,449 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting f50fbcc7323b4c76b740edd2e8a52a14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1733788650750 2024-12-09T23:57:36,449 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting d415a526ba3746289f01150529418575, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=552, earliestPutTs=1733788652975 2024-12-09T23:57:36,455 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71a18ca5bdcebe10547b57d8dd2e16f#C#compaction#149 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:57:36,455 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/83cd5b1bb32b48efac3058d1a92e50bf is 50, key is test_row_0/C:col10/1733788655132/Put/seqid=0 2024-12-09T23:57:36,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742004_1180 (size=12813) 2024-12-09T23:57:36,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T23:57:36,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:36,536 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing a71a18ca5bdcebe10547b57d8dd2e16f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T23:57:36,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=A 2024-12-09T23:57:36,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:36,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=B 2024-12-09T23:57:36,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:36,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=C 2024-12-09T23:57:36,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:36,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092cb23ac9f09e4aa3b6dc13a94399166d_a71a18ca5bdcebe10547b57d8dd2e16f is 50, key is test_row_0/A:col10/1733788655154/Put/seqid=0 2024-12-09T23:57:36,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742005_1181 (size=12454) 2024-12-09T23:57:36,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T23:57:36,842 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71a18ca5bdcebe10547b57d8dd2e16f#A#compaction#148 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:57:36,843 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/faa758617e6a47758d926232208d47d1 is 175, key is test_row_0/A:col10/1733788655132/Put/seqid=0 2024-12-09T23:57:36,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742006_1182 (size=31767) 2024-12-09T23:57:36,864 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/83cd5b1bb32b48efac3058d1a92e50bf as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/83cd5b1bb32b48efac3058d1a92e50bf 2024-12-09T23:57:36,868 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71a18ca5bdcebe10547b57d8dd2e16f/C of a71a18ca5bdcebe10547b57d8dd2e16f into 83cd5b1bb32b48efac3058d1a92e50bf(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:36,869 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71a18ca5bdcebe10547b57d8dd2e16f: 2024-12-09T23:57:36,869 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., storeName=a71a18ca5bdcebe10547b57d8dd2e16f/C, priority=13, startTime=1733788656404; duration=0sec 2024-12-09T23:57:36,871 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:36,871 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71a18ca5bdcebe10547b57d8dd2e16f:C 2024-12-09T23:57:36,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:36,951 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092cb23ac9f09e4aa3b6dc13a94399166d_a71a18ca5bdcebe10547b57d8dd2e16f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092cb23ac9f09e4aa3b6dc13a94399166d_a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:36,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/65630b85171443b3be61f4716a5973d7, store: [table=TestAcidGuarantees family=A region=a71a18ca5bdcebe10547b57d8dd2e16f] 2024-12-09T23:57:36,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/65630b85171443b3be61f4716a5973d7 is 175, key is test_row_0/A:col10/1733788655154/Put/seqid=0 2024-12-09T23:57:36,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742007_1183 (size=31255) 2024-12-09T23:57:37,251 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/faa758617e6a47758d926232208d47d1 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/faa758617e6a47758d926232208d47d1 2024-12-09T23:57:37,256 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71a18ca5bdcebe10547b57d8dd2e16f/A of a71a18ca5bdcebe10547b57d8dd2e16f into faa758617e6a47758d926232208d47d1(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:37,256 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71a18ca5bdcebe10547b57d8dd2e16f: 2024-12-09T23:57:37,256 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., storeName=a71a18ca5bdcebe10547b57d8dd2e16f/A, priority=13, startTime=1733788656404; duration=0sec 2024-12-09T23:57:37,256 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:37,256 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71a18ca5bdcebe10547b57d8dd2e16f:A 2024-12-09T23:57:37,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:37,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:37,358 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=565, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/65630b85171443b3be61f4716a5973d7 2024-12-09T23:57:37,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/32903d437def49919e0e26eadcf0385a is 50, key is test_row_0/B:col10/1733788655154/Put/seqid=0 2024-12-09T23:57:37,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:37,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 509 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788717365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:37,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:37,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:37,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:37,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742008_1184 (size=12301) 2024-12-09T23:57:37,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:37,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 510 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788717473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:37,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:37,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:37,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:37,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T23:57:37,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:37,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 511 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788717683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:37,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:37,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:37,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:37,772 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=565 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/32903d437def49919e0e26eadcf0385a 2024-12-09T23:57:37,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/09d27a6ea06b49e08cecb0e50779ae2a is 50, key is test_row_0/C:col10/1733788655154/Put/seqid=0 2024-12-09T23:57:37,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742009_1185 (size=12301) 2024-12-09T23:57:37,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:37,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 512 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788717993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:37,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:37,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:37,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:38,183 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=565 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/09d27a6ea06b49e08cecb0e50779ae2a 2024-12-09T23:57:38,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/65630b85171443b3be61f4716a5973d7 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/65630b85171443b3be61f4716a5973d7 2024-12-09T23:57:38,192 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/65630b85171443b3be61f4716a5973d7, entries=150, sequenceid=565, filesize=30.5 K 2024-12-09T23:57:38,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/32903d437def49919e0e26eadcf0385a as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/32903d437def49919e0e26eadcf0385a 2024-12-09T23:57:38,197 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/32903d437def49919e0e26eadcf0385a, entries=150, sequenceid=565, filesize=12.0 K 2024-12-09T23:57:38,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/09d27a6ea06b49e08cecb0e50779ae2a as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/09d27a6ea06b49e08cecb0e50779ae2a 2024-12-09T23:57:38,202 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/09d27a6ea06b49e08cecb0e50779ae2a, entries=150, sequenceid=565, filesize=12.0 K 2024-12-09T23:57:38,202 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a71a18ca5bdcebe10547b57d8dd2e16f in 1667ms, sequenceid=565, compaction requested=false 2024-12-09T23:57:38,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for a71a18ca5bdcebe10547b57d8dd2e16f: 2024-12-09T23:57:38,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:38,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-09T23:57:38,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-09T23:57:38,205 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-09T23:57:38,205 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7380 sec 2024-12-09T23:57:38,207 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.7430 sec 2024-12-09T23:57:38,292 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T23:57:38,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:38,505 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a71a18ca5bdcebe10547b57d8dd2e16f 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-09T23:57:38,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=A 2024-12-09T23:57:38,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:57:38,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=B 2024-12-09T23:57:38,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:57:38,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=C 2024-12-09T23:57:38,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:57:38,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209acb6d1dba2074b3bbab7b0673a6a323f_a71a18ca5bdcebe10547b57d8dd2e16f is 50, key is test_row_0/A:col10/1733788657363/Put/seqid=0 2024-12-09T23:57:38,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742010_1186 (size=14994) 2024-12-09T23:57:38,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:38,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 522 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788718531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:38,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:38,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:38,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:38,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:38,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 523 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788718643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:38,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:38,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:38,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:38,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:38,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 524 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788718852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:38,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:38,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:38,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:38,919 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:38,922 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209acb6d1dba2074b3bbab7b0673a6a323f_a71a18ca5bdcebe10547b57d8dd2e16f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209acb6d1dba2074b3bbab7b0673a6a323f_a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:38,923 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/1aee2315b6fc450cb458f1d458f06c09, store: [table=TestAcidGuarantees family=A region=a71a18ca5bdcebe10547b57d8dd2e16f] 2024-12-09T23:57:38,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/1aee2315b6fc450cb458f1d458f06c09 is 175, key is test_row_0/A:col10/1733788657363/Put/seqid=0 2024-12-09T23:57:38,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742011_1187 (size=39949) 2024-12-09T23:57:39,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:39,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 525 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788719163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:39,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:39,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:39,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:39,328 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=588, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/1aee2315b6fc450cb458f1d458f06c09 2024-12-09T23:57:39,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/68df3ee6ff194908bff58ac3c0c25738 is 50, key is test_row_0/B:col10/1733788657363/Put/seqid=0 2024-12-09T23:57:39,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742012_1188 (size=12301) 2024-12-09T23:57:39,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T23:57:39,633 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:57:39,634 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:57:39,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-09T23:57:39,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T23:57:39,635 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:57:39,636 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:57:39,636 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:57:39,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:39,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 526 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42780 deadline: 1733788719672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:39,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:39,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:39,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:57:39,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=588 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/68df3ee6ff194908bff58ac3c0c25738 2024-12-09T23:57:39,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T23:57:39,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/87da87afc53d4faf83fad0abe7cb5c09 is 50, key is test_row_0/C:col10/1733788657363/Put/seqid=0 2024-12-09T23:57:39,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742013_1189 (size=12301) 2024-12-09T23:57:39,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-09T23:57:39,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:39,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:39,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:39,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:39,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:39,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:39,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-09T23:57:39,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:39,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:39,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:39,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:39,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:39,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T23:57:40,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-09T23:57:40,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:40,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:40,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:40,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:40,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:40,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=588 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/87da87afc53d4faf83fad0abe7cb5c09 2024-12-09T23:57:40,177 INFO [Thread-612 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-612. 2024-12-09T23:57:40,177 DEBUG [Thread-612 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:57:40,177 DEBUG [Thread-612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:40,177 DEBUG [Thread-612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:40,177 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:57:40,179 INFO [Thread-610 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-610. 2024-12-09T23:57:40,179 DEBUG [Thread-610 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:57:40,179 DEBUG [Thread-610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:40,179 DEBUG [Thread-610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:40,179 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:57:40,180 INFO [Thread-608 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-608. 2024-12-09T23:57:40,180 DEBUG [Thread-608 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:57:40,180 DEBUG [Thread-608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:40,180 DEBUG [Thread-608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:40,180 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:57:40,182 INFO [Thread-614 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-614. 2024-12-09T23:57:40,182 DEBUG [Thread-614 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:57:40,182 DEBUG [Thread-614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:40,182 DEBUG [Thread-614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:40,182 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:57:40,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/1aee2315b6fc450cb458f1d458f06c09 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/1aee2315b6fc450cb458f1d458f06c09 2024-12-09T23:57:40,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/1aee2315b6fc450cb458f1d458f06c09, entries=200, sequenceid=588, filesize=39.0 K 2024-12-09T23:57:40,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/68df3ee6ff194908bff58ac3c0c25738 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/68df3ee6ff194908bff58ac3c0c25738 2024-12-09T23:57:40,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/68df3ee6ff194908bff58ac3c0c25738, entries=150, sequenceid=588, filesize=12.0 K 2024-12-09T23:57:40,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/87da87afc53d4faf83fad0abe7cb5c09 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/87da87afc53d4faf83fad0abe7cb5c09 2024-12-09T23:57:40,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/87da87afc53d4faf83fad0abe7cb5c09, entries=150, sequenceid=588, filesize=12.0 K 2024-12-09T23:57:40,217 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for a71a18ca5bdcebe10547b57d8dd2e16f in 1711ms, sequenceid=588, compaction requested=true 2024-12-09T23:57:40,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a71a18ca5bdcebe10547b57d8dd2e16f: 2024-12-09T23:57:40,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71a18ca5bdcebe10547b57d8dd2e16f:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:57:40,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:40,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71a18ca5bdcebe10547b57d8dd2e16f:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:57:40,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:57:40,217 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:57:40,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71a18ca5bdcebe10547b57d8dd2e16f:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:57:40,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:57:40,217 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:57:40,218 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102971 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:57:40,218 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:57:40,218 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): a71a18ca5bdcebe10547b57d8dd2e16f/A is initiating minor compaction (all files) 2024-12-09T23:57:40,218 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): a71a18ca5bdcebe10547b57d8dd2e16f/B is initiating minor compaction (all files) 2024-12-09T23:57:40,218 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71a18ca5bdcebe10547b57d8dd2e16f/A in TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:40,218 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71a18ca5bdcebe10547b57d8dd2e16f/B in TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:40,218 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/faa758617e6a47758d926232208d47d1, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/65630b85171443b3be61f4716a5973d7, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/1aee2315b6fc450cb458f1d458f06c09] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp, totalSize=100.6 K 2024-12-09T23:57:40,218 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/64adf21baa2b4395a6ffdda47cd491bf, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/32903d437def49919e0e26eadcf0385a, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/68df3ee6ff194908bff58ac3c0c25738] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp, totalSize=36.5 K 2024-12-09T23:57:40,218 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:40,218 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. files: [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/faa758617e6a47758d926232208d47d1, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/65630b85171443b3be61f4716a5973d7, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/1aee2315b6fc450cb458f1d458f06c09] 2024-12-09T23:57:40,218 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 64adf21baa2b4395a6ffdda47cd491bf, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=552, earliestPutTs=1733788652975 2024-12-09T23:57:40,219 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting faa758617e6a47758d926232208d47d1, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=552, earliestPutTs=1733788652975 2024-12-09T23:57:40,219 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 32903d437def49919e0e26eadcf0385a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=565, earliestPutTs=1733788655154 2024-12-09T23:57:40,219 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65630b85171443b3be61f4716a5973d7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=565, earliestPutTs=1733788655154 2024-12-09T23:57:40,219 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 68df3ee6ff194908bff58ac3c0c25738, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=588, earliestPutTs=1733788657356 2024-12-09T23:57:40,219 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1aee2315b6fc450cb458f1d458f06c09, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=588, earliestPutTs=1733788657355 2024-12-09T23:57:40,230 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71a18ca5bdcebe10547b57d8dd2e16f#B#compaction#156 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:57:40,230 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/32e3d487b1674e2ab3bd9209bb9d38f3 is 50, key is test_row_0/B:col10/1733788657363/Put/seqid=0 2024-12-09T23:57:40,231 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a71a18ca5bdcebe10547b57d8dd2e16f] 2024-12-09T23:57:40,234 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209f38a327e867444e18758e9ab041c8010_a71a18ca5bdcebe10547b57d8dd2e16f store=[table=TestAcidGuarantees family=A region=a71a18ca5bdcebe10547b57d8dd2e16f] 2024-12-09T23:57:40,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742014_1190 (size=12915) 2024-12-09T23:57:40,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-09T23:57:40,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:40,246 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing a71a18ca5bdcebe10547b57d8dd2e16f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T23:57:40,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=A 2024-12-09T23:57:40,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:40,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=B 2024-12-09T23:57:40,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:40,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=C 2024-12-09T23:57:40,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:40,255 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209f38a327e867444e18758e9ab041c8010_a71a18ca5bdcebe10547b57d8dd2e16f, store=[table=TestAcidGuarantees family=A region=a71a18ca5bdcebe10547b57d8dd2e16f] 2024-12-09T23:57:40,255 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f38a327e867444e18758e9ab041c8010_a71a18ca5bdcebe10547b57d8dd2e16f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a71a18ca5bdcebe10547b57d8dd2e16f] 2024-12-09T23:57:40,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d28990aadde74d6ca3d28682250b53ed_a71a18ca5bdcebe10547b57d8dd2e16f is 50, key is test_row_0/A:col10/1733788658530/Put/seqid=0 2024-12-09T23:57:40,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742015_1191 (size=4469) 2024-12-09T23:57:40,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T23:57:40,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742016_1192 (size=12454) 2024-12-09T23:57:40,648 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/32e3d487b1674e2ab3bd9209bb9d38f3 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/32e3d487b1674e2ab3bd9209bb9d38f3 2024-12-09T23:57:40,654 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71a18ca5bdcebe10547b57d8dd2e16f/B of a71a18ca5bdcebe10547b57d8dd2e16f into 32e3d487b1674e2ab3bd9209bb9d38f3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:40,654 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71a18ca5bdcebe10547b57d8dd2e16f: 2024-12-09T23:57:40,654 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., storeName=a71a18ca5bdcebe10547b57d8dd2e16f/B, priority=13, startTime=1733788660217; duration=0sec 2024-12-09T23:57:40,654 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:57:40,654 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71a18ca5bdcebe10547b57d8dd2e16f:B 2024-12-09T23:57:40,654 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:57:40,655 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:57:40,655 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): a71a18ca5bdcebe10547b57d8dd2e16f/C is initiating minor compaction (all files) 2024-12-09T23:57:40,655 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71a18ca5bdcebe10547b57d8dd2e16f/C in TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:40,655 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/83cd5b1bb32b48efac3058d1a92e50bf, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/09d27a6ea06b49e08cecb0e50779ae2a, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/87da87afc53d4faf83fad0abe7cb5c09] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp, totalSize=36.5 K 2024-12-09T23:57:40,656 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 83cd5b1bb32b48efac3058d1a92e50bf, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=552, earliestPutTs=1733788652975 2024-12-09T23:57:40,656 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 09d27a6ea06b49e08cecb0e50779ae2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=565, earliestPutTs=1733788655154 2024-12-09T23:57:40,657 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 87da87afc53d4faf83fad0abe7cb5c09, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=588, earliestPutTs=1733788657356 2024-12-09T23:57:40,661 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71a18ca5bdcebe10547b57d8dd2e16f#A#compaction#157 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:57:40,662 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/11afb0ee5fcb4873af27d58427a23ec2 is 175, key is test_row_0/A:col10/1733788657363/Put/seqid=0 2024-12-09T23:57:40,666 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71a18ca5bdcebe10547b57d8dd2e16f#C#compaction#159 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:57:40,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742017_1193 (size=31869) 2024-12-09T23:57:40,666 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/7bdc156411a4423e8241efd9ec141bd7 is 50, key is test_row_0/C:col10/1733788657363/Put/seqid=0 2024-12-09T23:57:40,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:40,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742018_1194 (size=12915) 2024-12-09T23:57:40,672 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d28990aadde74d6ca3d28682250b53ed_a71a18ca5bdcebe10547b57d8dd2e16f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d28990aadde74d6ca3d28682250b53ed_a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:40,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/357e10c5bc5c483fbd9581f48867a6d1, store: [table=TestAcidGuarantees family=A region=a71a18ca5bdcebe10547b57d8dd2e16f] 2024-12-09T23:57:40,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/357e10c5bc5c483fbd9581f48867a6d1 is 175, key is test_row_0/A:col10/1733788658530/Put/seqid=0 2024-12-09T23:57:40,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742019_1195 (size=31255) 2024-12-09T23:57:40,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:40,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. as already flushing 2024-12-09T23:57:40,692 INFO [Thread-603 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-603. 2024-12-09T23:57:40,692 DEBUG [Thread-603 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:57:40,692 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:40,692 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:40,692 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:57:40,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T23:57:41,078 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/11afb0ee5fcb4873af27d58427a23ec2 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/11afb0ee5fcb4873af27d58427a23ec2 2024-12-09T23:57:41,078 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=599, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/357e10c5bc5c483fbd9581f48867a6d1 2024-12-09T23:57:41,080 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/7bdc156411a4423e8241efd9ec141bd7 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/7bdc156411a4423e8241efd9ec141bd7 2024-12-09T23:57:41,085 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71a18ca5bdcebe10547b57d8dd2e16f/A of a71a18ca5bdcebe10547b57d8dd2e16f into 11afb0ee5fcb4873af27d58427a23ec2(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:41,085 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71a18ca5bdcebe10547b57d8dd2e16f: 2024-12-09T23:57:41,085 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., storeName=a71a18ca5bdcebe10547b57d8dd2e16f/A, priority=13, startTime=1733788660217; duration=0sec 2024-12-09T23:57:41,085 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:41,085 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71a18ca5bdcebe10547b57d8dd2e16f:A 2024-12-09T23:57:41,086 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71a18ca5bdcebe10547b57d8dd2e16f/C of a71a18ca5bdcebe10547b57d8dd2e16f into 7bdc156411a4423e8241efd9ec141bd7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:41,086 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71a18ca5bdcebe10547b57d8dd2e16f: 2024-12-09T23:57:41,086 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., storeName=a71a18ca5bdcebe10547b57d8dd2e16f/C, priority=13, startTime=1733788660217; duration=0sec 2024-12-09T23:57:41,086 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:41,086 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71a18ca5bdcebe10547b57d8dd2e16f:C 2024-12-09T23:57:41,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/4c1a9d0029bd4aa289d2cfa4a0672086 is 50, key is test_row_0/B:col10/1733788658530/Put/seqid=0 2024-12-09T23:57:41,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742020_1196 (size=12301) 2024-12-09T23:57:41,327 INFO [Thread-597 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-597. 2024-12-09T23:57:41,328 DEBUG [Thread-597 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:57:41,328 DEBUG [Thread-597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:41,328 DEBUG [Thread-597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:41,328 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:57:41,375 INFO [Thread-599 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-599. 2024-12-09T23:57:41,375 DEBUG [Thread-599 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:57:41,375 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:41,375 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:41,375 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:57:41,397 INFO [Thread-605 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-605. 2024-12-09T23:57:41,397 INFO [Thread-601 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-601. 2024-12-09T23:57:41,397 DEBUG [Thread-605 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:57:41,397 DEBUG [Thread-601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:57:41,397 DEBUG [Thread-605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:41,397 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:41,397 DEBUG [Thread-605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:41,397 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:41,397 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:57:41,397 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:57:41,493 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/4c1a9d0029bd4aa289d2cfa4a0672086 2024-12-09T23:57:41,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/cd4f9a31e05c481fb68d2b6aba2117f9 is 50, key is test_row_0/C:col10/1733788658530/Put/seqid=0 2024-12-09T23:57:41,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742021_1197 (size=12301) 2024-12-09T23:57:41,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T23:57:41,910 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/cd4f9a31e05c481fb68d2b6aba2117f9 2024-12-09T23:57:41,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/357e10c5bc5c483fbd9581f48867a6d1 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/357e10c5bc5c483fbd9581f48867a6d1 2024-12-09T23:57:41,925 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/357e10c5bc5c483fbd9581f48867a6d1, entries=150, sequenceid=599, filesize=30.5 K 2024-12-09T23:57:41,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/4c1a9d0029bd4aa289d2cfa4a0672086 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/4c1a9d0029bd4aa289d2cfa4a0672086 2024-12-09T23:57:41,930 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/4c1a9d0029bd4aa289d2cfa4a0672086, entries=150, sequenceid=599, filesize=12.0 K 2024-12-09T23:57:41,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/cd4f9a31e05c481fb68d2b6aba2117f9 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/cd4f9a31e05c481fb68d2b6aba2117f9 2024-12-09T23:57:41,935 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/cd4f9a31e05c481fb68d2b6aba2117f9, entries=150, sequenceid=599, filesize=12.0 K 2024-12-09T23:57:41,936 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=33.54 KB/34350 for a71a18ca5bdcebe10547b57d8dd2e16f in 1690ms, sequenceid=599, compaction requested=false 2024-12-09T23:57:41,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for a71a18ca5bdcebe10547b57d8dd2e16f: 2024-12-09T23:57:41,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:41,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-09T23:57:41,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-09T23:57:41,939 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-09T23:57:41,939 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3010 sec 2024-12-09T23:57:41,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.3060 sec 2024-12-09T23:57:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T23:57:43,815 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:57:43,815 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T23:57:43,816 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-12-09T23:57:43,816 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-12-09T23:57:43,816 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-12-09T23:57:43,816 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 483 2024-12-09T23:57:43,816 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-12-09T23:57:43,816 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T23:57:43,816 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4605 2024-12-09T23:57:43,816 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4473 2024-12-09T23:57:43,816 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T23:57:43,816 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2023 2024-12-09T23:57:43,816 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 6066 rows 2024-12-09T23:57:43,816 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2018 2024-12-09T23:57:43,816 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 6054 rows 2024-12-09T23:57:43,816 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T23:57:43,817 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T23:57:43,817 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:43,817 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:43,817 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:57:43,820 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-09T23:57:43,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T23:57:43,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T23:57:43,825 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733788663825"}]},"ts":"1733788663825"} 2024-12-09T23:57:43,827 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T23:57:43,827 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T23:57:43,828 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T23:57:43,829 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71a18ca5bdcebe10547b57d8dd2e16f, UNASSIGN}] 2024-12-09T23:57:43,830 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71a18ca5bdcebe10547b57d8dd2e16f, UNASSIGN 2024-12-09T23:57:43,831 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=a71a18ca5bdcebe10547b57d8dd2e16f, regionState=CLOSING, regionLocation=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:43,832 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71a18ca5bdcebe10547b57d8dd2e16f, UNASSIGN because future has completed 2024-12-09T23:57:43,832 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T23:57:43,832 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837}] 2024-12-09T23:57:43,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T23:57:43,986 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(122): Close a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:43,986 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T23:57:43,986 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1722): Closing a71a18ca5bdcebe10547b57d8dd2e16f, disabling compactions & flushes 2024-12-09T23:57:43,986 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:43,986 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:43,986 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. after waiting 0 ms 2024-12-09T23:57:43,986 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:43,987 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(2902): Flushing a71a18ca5bdcebe10547b57d8dd2e16f 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-09T23:57:43,987 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=A 2024-12-09T23:57:43,987 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:43,988 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=B 2024-12-09T23:57:43,988 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:43,988 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71a18ca5bdcebe10547b57d8dd2e16f, store=C 2024-12-09T23:57:43,988 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:43,998 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412096c978bee29a14d73ba4550a39c9a5ec9_a71a18ca5bdcebe10547b57d8dd2e16f is 50, key is test_row_0/A:col10/1733788661394/Put/seqid=0 2024-12-09T23:57:44,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742022_1198 (size=12454) 2024-12-09T23:57:44,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T23:57:44,404 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:44,414 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412096c978bee29a14d73ba4550a39c9a5ec9_a71a18ca5bdcebe10547b57d8dd2e16f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096c978bee29a14d73ba4550a39c9a5ec9_a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:44,416 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/56414a1228854895bf48179dc0a9f748, store: [table=TestAcidGuarantees family=A region=a71a18ca5bdcebe10547b57d8dd2e16f] 2024-12-09T23:57:44,417 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/56414a1228854895bf48179dc0a9f748 is 175, key is test_row_0/A:col10/1733788661394/Put/seqid=0 2024-12-09T23:57:44,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742023_1199 (size=31255) 2024-12-09T23:57:44,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T23:57:44,825 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=610, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/56414a1228854895bf48179dc0a9f748 2024-12-09T23:57:44,838 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/f71866f6ea144e95be761df66ec3cdaf is 50, key is test_row_0/B:col10/1733788661394/Put/seqid=0 2024-12-09T23:57:44,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742024_1200 (size=12301) 2024-12-09T23:57:44,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T23:57:45,244 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=610 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/f71866f6ea144e95be761df66ec3cdaf 2024-12-09T23:57:45,303 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/7d1225ce375f418a9649bf70a86128c0 is 50, key is test_row_0/C:col10/1733788661394/Put/seqid=0 2024-12-09T23:57:45,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742025_1201 (size=12301) 2024-12-09T23:57:45,709 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=610 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/7d1225ce375f418a9649bf70a86128c0 2024-12-09T23:57:45,718 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/A/56414a1228854895bf48179dc0a9f748 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/56414a1228854895bf48179dc0a9f748 2024-12-09T23:57:45,725 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/56414a1228854895bf48179dc0a9f748, entries=150, sequenceid=610, filesize=30.5 K 2024-12-09T23:57:45,726 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/B/f71866f6ea144e95be761df66ec3cdaf as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/f71866f6ea144e95be761df66ec3cdaf 2024-12-09T23:57:45,731 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/f71866f6ea144e95be761df66ec3cdaf, entries=150, sequenceid=610, filesize=12.0 K 2024-12-09T23:57:45,732 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/.tmp/C/7d1225ce375f418a9649bf70a86128c0 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/7d1225ce375f418a9649bf70a86128c0 2024-12-09T23:57:45,736 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/7d1225ce375f418a9649bf70a86128c0, entries=150, sequenceid=610, filesize=12.0 K 2024-12-09T23:57:45,737 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for a71a18ca5bdcebe10547b57d8dd2e16f in 1751ms, sequenceid=610, compaction requested=true 2024-12-09T23:57:45,737 DEBUG [StoreCloser-TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/00bddf5f99694eb0930998dd074bc90b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/52ee60f59e134e279a554f58c429422c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/8569d0ba2e9c40f0b5400accf3f2ae50, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/1b64e92846b840ebb974392db1ce0c98, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/bad9ee60fcc64d219c05104c7e2099dd, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/6af81dbb28d14245899f9292311317b8, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/827cc18d1f844488bf01c965b05f4ad8, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/d25b341c859547b9bb52e351f1a69f09, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/92dc398ac9ba4bbb93cb2d4b9538ff53, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/a849eac66caa445cb7926ef2e07efcf1, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/c95153df518c4a86ae08056e24e8361e, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/4650fa800fb849599c6bdaac77d92b1d, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/bc66703e1b7249b39605fb65f922abe1, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/3e20fa792dc3420db4ee60916ea43146, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/fb772c8b7fe24e1f83e1f82c88211d48, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/faa758617e6a47758d926232208d47d1, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/65630b85171443b3be61f4716a5973d7, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/1aee2315b6fc450cb458f1d458f06c09] to archive 2024-12-09T23:57:45,738 DEBUG [StoreCloser-TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-09T23:57:45,741 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/00bddf5f99694eb0930998dd074bc90b to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/00bddf5f99694eb0930998dd074bc90b 2024-12-09T23:57:45,741 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/52ee60f59e134e279a554f58c429422c to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/52ee60f59e134e279a554f58c429422c 2024-12-09T23:57:45,741 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/827cc18d1f844488bf01c965b05f4ad8 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/827cc18d1f844488bf01c965b05f4ad8 2024-12-09T23:57:45,741 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/6af81dbb28d14245899f9292311317b8 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/6af81dbb28d14245899f9292311317b8 2024-12-09T23:57:45,741 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/bad9ee60fcc64d219c05104c7e2099dd to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/bad9ee60fcc64d219c05104c7e2099dd 2024-12-09T23:57:45,741 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/d25b341c859547b9bb52e351f1a69f09 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/d25b341c859547b9bb52e351f1a69f09 2024-12-09T23:57:45,742 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/8569d0ba2e9c40f0b5400accf3f2ae50 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/8569d0ba2e9c40f0b5400accf3f2ae50 2024-12-09T23:57:45,742 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/1b64e92846b840ebb974392db1ce0c98 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/1b64e92846b840ebb974392db1ce0c98 2024-12-09T23:57:45,744 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/92dc398ac9ba4bbb93cb2d4b9538ff53 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/92dc398ac9ba4bbb93cb2d4b9538ff53 2024-12-09T23:57:45,744 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/a849eac66caa445cb7926ef2e07efcf1 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/a849eac66caa445cb7926ef2e07efcf1 2024-12-09T23:57:45,744 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/c95153df518c4a86ae08056e24e8361e to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/c95153df518c4a86ae08056e24e8361e 2024-12-09T23:57:45,744 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/4650fa800fb849599c6bdaac77d92b1d to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/4650fa800fb849599c6bdaac77d92b1d 2024-12-09T23:57:45,745 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/bc66703e1b7249b39605fb65f922abe1 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/bc66703e1b7249b39605fb65f922abe1 2024-12-09T23:57:45,745 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/fb772c8b7fe24e1f83e1f82c88211d48 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/fb772c8b7fe24e1f83e1f82c88211d48 2024-12-09T23:57:45,745 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/3e20fa792dc3420db4ee60916ea43146 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/3e20fa792dc3420db4ee60916ea43146 2024-12-09T23:57:45,745 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/faa758617e6a47758d926232208d47d1 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/faa758617e6a47758d926232208d47d1 2024-12-09T23:57:45,746 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/65630b85171443b3be61f4716a5973d7 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/65630b85171443b3be61f4716a5973d7 2024-12-09T23:57:45,746 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/1aee2315b6fc450cb458f1d458f06c09 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/1aee2315b6fc450cb458f1d458f06c09 2024-12-09T23:57:45,747 DEBUG [StoreCloser-TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/bf45d9eb798e450cb8dd58ea9721390b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/34e753f926594af39f8d05c11e4d11c4, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/21693dea849c47269b076b380514a9c8, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/2b3eca773ce74b388677c014693d0589, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/2089556f6d3e49d49fcd36d96b9d4394, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/1eb125861cc44563abe93d4925a8d20a, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/b11fe4f3cd78426f82d647655e9cabf8, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/952ed4855b7044b69fd980235fdf0fd3, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/3b17060f2e0942238e17b8a5bcb382df, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/f0da2c21a16843ceadabb04044a80048, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/af2060c672074ab99f905e7f60dd5f2f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/ba1bd1c900eb49748019ee06fe776c8c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/b0e693f4e46c47fcabc78df326a8dac7, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/a5393b3abd0b4c619c05d06f15a66044, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/64adf21baa2b4395a6ffdda47cd491bf, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/59580fa685b5421ea585c288af558d1e, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/32903d437def49919e0e26eadcf0385a, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/68df3ee6ff194908bff58ac3c0c25738] to archive 2024-12-09T23:57:45,748 DEBUG [StoreCloser-TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-09T23:57:45,750 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/34e753f926594af39f8d05c11e4d11c4 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/34e753f926594af39f8d05c11e4d11c4 2024-12-09T23:57:45,750 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/21693dea849c47269b076b380514a9c8 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/21693dea849c47269b076b380514a9c8 2024-12-09T23:57:45,750 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/952ed4855b7044b69fd980235fdf0fd3 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/952ed4855b7044b69fd980235fdf0fd3 2024-12-09T23:57:45,750 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/2b3eca773ce74b388677c014693d0589 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/2b3eca773ce74b388677c014693d0589 2024-12-09T23:57:45,751 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/bf45d9eb798e450cb8dd58ea9721390b to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/bf45d9eb798e450cb8dd58ea9721390b 2024-12-09T23:57:45,751 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/1eb125861cc44563abe93d4925a8d20a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/1eb125861cc44563abe93d4925a8d20a 2024-12-09T23:57:45,751 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/2089556f6d3e49d49fcd36d96b9d4394 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/2089556f6d3e49d49fcd36d96b9d4394 2024-12-09T23:57:45,751 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/b11fe4f3cd78426f82d647655e9cabf8 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/b11fe4f3cd78426f82d647655e9cabf8 2024-12-09T23:57:45,752 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/3b17060f2e0942238e17b8a5bcb382df to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/3b17060f2e0942238e17b8a5bcb382df 2024-12-09T23:57:45,752 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/af2060c672074ab99f905e7f60dd5f2f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/af2060c672074ab99f905e7f60dd5f2f 2024-12-09T23:57:45,752 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/f0da2c21a16843ceadabb04044a80048 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/f0da2c21a16843ceadabb04044a80048 2024-12-09T23:57:45,752 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/ba1bd1c900eb49748019ee06fe776c8c to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/ba1bd1c900eb49748019ee06fe776c8c 2024-12-09T23:57:45,752 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/b0e693f4e46c47fcabc78df326a8dac7 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/b0e693f4e46c47fcabc78df326a8dac7 2024-12-09T23:57:45,753 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/a5393b3abd0b4c619c05d06f15a66044 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/a5393b3abd0b4c619c05d06f15a66044 2024-12-09T23:57:45,753 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/64adf21baa2b4395a6ffdda47cd491bf to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/64adf21baa2b4395a6ffdda47cd491bf 2024-12-09T23:57:45,753 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/59580fa685b5421ea585c288af558d1e to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/59580fa685b5421ea585c288af558d1e 2024-12-09T23:57:45,753 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/32903d437def49919e0e26eadcf0385a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/32903d437def49919e0e26eadcf0385a 2024-12-09T23:57:45,753 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/68df3ee6ff194908bff58ac3c0c25738 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/68df3ee6ff194908bff58ac3c0c25738 2024-12-09T23:57:45,755 DEBUG [StoreCloser-TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/9d73cdc7408449d28210569edda95ad4, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/9ef085b8d190434dbe7362d8cc204a7b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/9bbdf3bd493848d1ab59fcf87469c32c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/d9e522cfb45542e2a953d45a0b06df8f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/d2b8b3df0658468c829f4f793323dc64, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/c9da8494ebb64820956c08bbcb29104c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/fcc07f32e5f24b33b8cb5a832b31846e, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/e8ba0aeadf3a47cbb3a0ab28a369049e, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/a27a61b5e9f7463c9e7999a53dd35a83, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/d500431bd2574af897271c5ee43d73ec, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/b7006eca5c4b4d5384faa8f7ad8dc6c8, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/5245319cc8e842c2ad6556172f106b3e, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/4db13686f5f14bff9874c3c358508621, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/f50fbcc7323b4c76b740edd2e8a52a14, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/83cd5b1bb32b48efac3058d1a92e50bf, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/d415a526ba3746289f01150529418575, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/09d27a6ea06b49e08cecb0e50779ae2a, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/87da87afc53d4faf83fad0abe7cb5c09] to archive 2024-12-09T23:57:45,755 DEBUG [StoreCloser-TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-09T23:57:45,757 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/9ef085b8d190434dbe7362d8cc204a7b to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/9ef085b8d190434dbe7362d8cc204a7b 2024-12-09T23:57:45,758 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/d2b8b3df0658468c829f4f793323dc64 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/d2b8b3df0658468c829f4f793323dc64 2024-12-09T23:57:45,758 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/9bbdf3bd493848d1ab59fcf87469c32c to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/9bbdf3bd493848d1ab59fcf87469c32c 2024-12-09T23:57:45,758 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/e8ba0aeadf3a47cbb3a0ab28a369049e to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/e8ba0aeadf3a47cbb3a0ab28a369049e 2024-12-09T23:57:45,758 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/9d73cdc7408449d28210569edda95ad4 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/9d73cdc7408449d28210569edda95ad4 2024-12-09T23:57:45,758 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/c9da8494ebb64820956c08bbcb29104c to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/c9da8494ebb64820956c08bbcb29104c 2024-12-09T23:57:45,758 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/fcc07f32e5f24b33b8cb5a832b31846e to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/fcc07f32e5f24b33b8cb5a832b31846e 2024-12-09T23:57:45,758 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/d9e522cfb45542e2a953d45a0b06df8f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/d9e522cfb45542e2a953d45a0b06df8f 2024-12-09T23:57:45,759 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/d500431bd2574af897271c5ee43d73ec to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/d500431bd2574af897271c5ee43d73ec 2024-12-09T23:57:45,759 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/5245319cc8e842c2ad6556172f106b3e to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/5245319cc8e842c2ad6556172f106b3e 2024-12-09T23:57:45,760 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/b7006eca5c4b4d5384faa8f7ad8dc6c8 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/b7006eca5c4b4d5384faa8f7ad8dc6c8 2024-12-09T23:57:45,760 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/a27a61b5e9f7463c9e7999a53dd35a83 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/a27a61b5e9f7463c9e7999a53dd35a83 2024-12-09T23:57:45,760 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/83cd5b1bb32b48efac3058d1a92e50bf to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/83cd5b1bb32b48efac3058d1a92e50bf 2024-12-09T23:57:45,760 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/4db13686f5f14bff9874c3c358508621 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/4db13686f5f14bff9874c3c358508621 2024-12-09T23:57:45,760 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/f50fbcc7323b4c76b740edd2e8a52a14 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/f50fbcc7323b4c76b740edd2e8a52a14 2024-12-09T23:57:45,760 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/d415a526ba3746289f01150529418575 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/d415a526ba3746289f01150529418575 2024-12-09T23:57:45,760 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/87da87afc53d4faf83fad0abe7cb5c09 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/87da87afc53d4faf83fad0abe7cb5c09 2024-12-09T23:57:45,761 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/09d27a6ea06b49e08cecb0e50779ae2a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/09d27a6ea06b49e08cecb0e50779ae2a 2024-12-09T23:57:45,765 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/recovered.edits/613.seqid, newMaxSeqId=613, maxSeqId=4 2024-12-09T23:57:45,765 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. 2024-12-09T23:57:45,766 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1676): Region close journal for a71a18ca5bdcebe10547b57d8dd2e16f: Waiting for close lock at 1733788663986Running coprocessor pre-close hooks at 1733788663986Disabling compacts and flushes for region at 1733788663986Disabling writes for close at 1733788663986Obtaining lock to block concurrent updates at 1733788663987 (+1 ms)Preparing flush snapshotting stores in a71a18ca5bdcebe10547b57d8dd2e16f at 1733788663987Finished memstore snapshotting TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733788663988 (+1 ms)Flushing stores of TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f. at 1733788663989 (+1 ms)Flushing a71a18ca5bdcebe10547b57d8dd2e16f/A: creating writer at 1733788663989Flushing mob file a71a18ca5bdcebe10547b57d8dd2e16f/A: appending metadata at 1733788663998 (+9 ms)Flushing mob file a71a18ca5bdcebe10547b57d8dd2e16f/A: closing flushed file at 1733788663998Flushing a71a18ca5bdcebe10547b57d8dd2e16f/A: appending metadata at 1733788664416 (+418 ms)Flushing a71a18ca5bdcebe10547b57d8dd2e16f/A: closing flushed file at 1733788664416Flushing a71a18ca5bdcebe10547b57d8dd2e16f/B: creating writer at 1733788664834 (+418 ms)Flushing a71a18ca5bdcebe10547b57d8dd2e16f/B: appending metadata at 1733788664838 (+4 ms)Flushing a71a18ca5bdcebe10547b57d8dd2e16f/B: closing flushed file at 1733788664838Flushing a71a18ca5bdcebe10547b57d8dd2e16f/C: creating writer at 1733788665294 (+456 ms)Flushing a71a18ca5bdcebe10547b57d8dd2e16f/C: appending metadata at 1733788665301 (+7 ms)Flushing a71a18ca5bdcebe10547b57d8dd2e16f/C: closing flushed file at 1733788665301Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7fea6a33: reopening flushed file at 1733788665717 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22f814af: reopening flushed file at 1733788665725 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9f07d9b: reopening flushed file at 1733788665731 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for a71a18ca5bdcebe10547b57d8dd2e16f in 1751ms, sequenceid=610, compaction requested=true at 1733788665737 (+6 ms)Writing region close event to WAL at 1733788665762 (+25 ms)Running coprocessor post-close hooks at 1733788665765 (+3 ms)Closed at 1733788665765 2024-12-09T23:57:45,768 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(157): Closed a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:45,768 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=a71a18ca5bdcebe10547b57d8dd2e16f, regionState=CLOSED 2024-12-09T23:57:45,769 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 because future has completed 2024-12-09T23:57:45,771 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-09T23:57:45,771 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseRegionProcedure a71a18ca5bdcebe10547b57d8dd2e16f, server=2907d75fbb3e,37627,1733788599837 in 1.9380 sec 2024-12-09T23:57:45,772 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-12-09T23:57:45,773 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71a18ca5bdcebe10547b57d8dd2e16f, UNASSIGN in 1.9420 sec 2024-12-09T23:57:45,775 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-09T23:57:45,775 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9450 sec 2024-12-09T23:57:45,776 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733788665776"}]},"ts":"1733788665776"} 2024-12-09T23:57:45,777 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T23:57:45,777 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T23:57:45,778 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9570 sec 2024-12-09T23:57:45,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T23:57:45,973 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:57:45,974 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-09T23:57:45,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T23:57:45,976 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T23:57:45,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-09T23:57:45,977 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=56, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T23:57:45,980 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:45,985 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A, FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B, FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C, FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/recovered.edits] 2024-12-09T23:57:45,989 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/11afb0ee5fcb4873af27d58427a23ec2 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/11afb0ee5fcb4873af27d58427a23ec2 2024-12-09T23:57:45,989 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/357e10c5bc5c483fbd9581f48867a6d1 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/357e10c5bc5c483fbd9581f48867a6d1 2024-12-09T23:57:45,989 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/56414a1228854895bf48179dc0a9f748 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/A/56414a1228854895bf48179dc0a9f748 2024-12-09T23:57:45,992 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/32e3d487b1674e2ab3bd9209bb9d38f3 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/32e3d487b1674e2ab3bd9209bb9d38f3 2024-12-09T23:57:45,992 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/f71866f6ea144e95be761df66ec3cdaf to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/f71866f6ea144e95be761df66ec3cdaf 2024-12-09T23:57:45,992 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/4c1a9d0029bd4aa289d2cfa4a0672086 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/B/4c1a9d0029bd4aa289d2cfa4a0672086 2024-12-09T23:57:45,995 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/7d1225ce375f418a9649bf70a86128c0 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/7d1225ce375f418a9649bf70a86128c0 2024-12-09T23:57:45,995 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/7bdc156411a4423e8241efd9ec141bd7 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/7bdc156411a4423e8241efd9ec141bd7 2024-12-09T23:57:45,995 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/cd4f9a31e05c481fb68d2b6aba2117f9 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/C/cd4f9a31e05c481fb68d2b6aba2117f9 2024-12-09T23:57:45,997 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/recovered.edits/613.seqid to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f/recovered.edits/613.seqid 2024-12-09T23:57:45,997 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:45,997 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T23:57:45,998 DEBUG [PEWorker-3 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T23:57:45,998 DEBUG [PEWorker-3 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-09T23:57:46,007 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091b41d339a676480e9e087a6ba319a674_a71a18ca5bdcebe10547b57d8dd2e16f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091b41d339a676480e9e087a6ba319a674_a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:46,007 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209256753eea19d44d79889e9a8eb0c7015_a71a18ca5bdcebe10547b57d8dd2e16f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209256753eea19d44d79889e9a8eb0c7015_a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:46,007 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092cb23ac9f09e4aa3b6dc13a94399166d_a71a18ca5bdcebe10547b57d8dd2e16f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092cb23ac9f09e4aa3b6dc13a94399166d_a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:46,008 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412090a3a72e2166040a9b42e7e5dfc690564_a71a18ca5bdcebe10547b57d8dd2e16f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412090a3a72e2166040a9b42e7e5dfc690564_a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:46,008 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096c978bee29a14d73ba4550a39c9a5ec9_a71a18ca5bdcebe10547b57d8dd2e16f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096c978bee29a14d73ba4550a39c9a5ec9_a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:46,008 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412097cf7711584604ce28844dcd0cd2a76c8_a71a18ca5bdcebe10547b57d8dd2e16f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412097cf7711584604ce28844dcd0cd2a76c8_a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:46,008 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209730121f34493464382630a1d989433be_a71a18ca5bdcebe10547b57d8dd2e16f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209730121f34493464382630a1d989433be_a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:46,008 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a090ea079782460e9f9430bf86de32ea_a71a18ca5bdcebe10547b57d8dd2e16f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a090ea079782460e9f9430bf86de32ea_a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:46,009 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e01bafe02617476aad804836631b8e84_a71a18ca5bdcebe10547b57d8dd2e16f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e01bafe02617476aad804836631b8e84_a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:46,010 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d28990aadde74d6ca3d28682250b53ed_a71a18ca5bdcebe10547b57d8dd2e16f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d28990aadde74d6ca3d28682250b53ed_a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:46,010 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d7a33a2f51fb46ca9467ea82d75fa908_a71a18ca5bdcebe10547b57d8dd2e16f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d7a33a2f51fb46ca9467ea82d75fa908_a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:46,010 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b7d585851d194f0e996dd3db91c7d55b_a71a18ca5bdcebe10547b57d8dd2e16f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b7d585851d194f0e996dd3db91c7d55b_a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:46,010 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d5fcebde4e994e04a86709488fb3e901_a71a18ca5bdcebe10547b57d8dd2e16f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d5fcebde4e994e04a86709488fb3e901_a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:46,010 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209acb6d1dba2074b3bbab7b0673a6a323f_a71a18ca5bdcebe10547b57d8dd2e16f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209acb6d1dba2074b3bbab7b0673a6a323f_a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:46,010 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ebf9aa6026994afbafa1d02cf925307d_a71a18ca5bdcebe10547b57d8dd2e16f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ebf9aa6026994afbafa1d02cf925307d_a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:46,011 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f81ced5ba68c4913a5049dcb5251c405_a71a18ca5bdcebe10547b57d8dd2e16f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f81ced5ba68c4913a5049dcb5251c405_a71a18ca5bdcebe10547b57d8dd2e16f 2024-12-09T23:57:46,011 DEBUG [PEWorker-3 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T23:57:46,013 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=56, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T23:57:46,015 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T23:57:46,017 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T23:57:46,018 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=56, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T23:57:46,018 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-09T23:57:46,018 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733788666018"}]},"ts":"9223372036854775807"} 2024-12-09T23:57:46,020 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T23:57:46,020 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a71a18ca5bdcebe10547b57d8dd2e16f, NAME => 'TestAcidGuarantees,,1733788636926.a71a18ca5bdcebe10547b57d8dd2e16f.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T23:57:46,020 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T23:57:46,020 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733788666020"}]},"ts":"9223372036854775807"} 2024-12-09T23:57:46,021 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T23:57:46,022 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=56, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T23:57:46,023 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 48 msec 2024-12-09T23:57:46,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-09T23:57:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T23:57:46,089 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:57:46,100 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=258 (was 254) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8bc2ec3c-7335-1808-2d8f-cd0f163ec5b4/cluster_96f75c57-c245-065f-99c2-7563c90f684d/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1230264142_22 at /127.0.0.1:51178 [Waiting for operation #412] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8bc2ec3c-7335-1808-2d8f-cd0f163ec5b4/cluster_96f75c57-c245-065f-99c2-7563c90f684d/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1230264142_22 at /127.0.0.1:51546 [Waiting for operation #326] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1827791340_22 at /127.0.0.1:51544 [Waiting for operation #328] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1827791340_22 at /127.0.0.1:34300 [Waiting for operation #438] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=375 (was 333) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4029 (was 4115) 2024-12-09T23:57:46,111 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=258, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=375, ProcessCount=11, AvailableMemoryMB=4028 2024-12-09T23:57:46,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T23:57:46,113 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T23:57:46,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T23:57:46,116 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T23:57:46,116 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:46,116 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 57 2024-12-09T23:57:46,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-09T23:57:46,117 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T23:57:46,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742026_1202 (size=1061) 2024-12-09T23:57:46,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-09T23:57:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-09T23:57:46,530 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e 2024-12-09T23:57:46,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742027_1203 (size=53) 2024-12-09T23:57:46,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-09T23:57:46,939 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:57:46,939 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 472b3103632c40151a72730c972c9301, disabling compactions & flushes 2024-12-09T23:57:46,939 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:46,939 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:46,939 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. after waiting 0 ms 2024-12-09T23:57:46,939 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:46,939 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:46,939 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 472b3103632c40151a72730c972c9301: Waiting for close lock at 1733788666939Disabling compacts and flushes for region at 1733788666939Disabling writes for close at 1733788666939Writing region close event to WAL at 1733788666939Closed at 1733788666939 2024-12-09T23:57:46,941 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T23:57:46,941 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733788666941"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733788666941"}]},"ts":"1733788666941"} 2024-12-09T23:57:46,944 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T23:57:46,945 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T23:57:46,945 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733788666945"}]},"ts":"1733788666945"} 2024-12-09T23:57:46,948 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T23:57:46,948 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=472b3103632c40151a72730c972c9301, ASSIGN}] 2024-12-09T23:57:46,951 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=472b3103632c40151a72730c972c9301, ASSIGN 2024-12-09T23:57:46,951 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=472b3103632c40151a72730c972c9301, ASSIGN; state=OFFLINE, location=2907d75fbb3e,37627,1733788599837; forceNewPlan=false, retain=false 2024-12-09T23:57:47,103 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=472b3103632c40151a72730c972c9301, regionState=OPENING, regionLocation=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:47,108 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=472b3103632c40151a72730c972c9301, ASSIGN because future has completed 2024-12-09T23:57:47,109 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure 472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837}] 2024-12-09T23:57:47,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-09T23:57:47,270 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:47,271 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} 2024-12-09T23:57:47,272 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 472b3103632c40151a72730c972c9301 2024-12-09T23:57:47,272 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:57:47,272 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for 472b3103632c40151a72730c972c9301 2024-12-09T23:57:47,272 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for 472b3103632c40151a72730c972c9301 2024-12-09T23:57:47,275 INFO [StoreOpener-472b3103632c40151a72730c972c9301-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 472b3103632c40151a72730c972c9301 2024-12-09T23:57:47,277 INFO [StoreOpener-472b3103632c40151a72730c972c9301-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T23:57:47,277 INFO [StoreOpener-472b3103632c40151a72730c972c9301-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 472b3103632c40151a72730c972c9301 columnFamilyName A 2024-12-09T23:57:47,277 DEBUG [StoreOpener-472b3103632c40151a72730c972c9301-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:47,278 INFO [StoreOpener-472b3103632c40151a72730c972c9301-1 {}] regionserver.HStore(327): Store=472b3103632c40151a72730c972c9301/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:57:47,278 INFO [StoreOpener-472b3103632c40151a72730c972c9301-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 472b3103632c40151a72730c972c9301 2024-12-09T23:57:47,279 INFO [StoreOpener-472b3103632c40151a72730c972c9301-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T23:57:47,280 INFO [StoreOpener-472b3103632c40151a72730c972c9301-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 472b3103632c40151a72730c972c9301 columnFamilyName B 2024-12-09T23:57:47,280 DEBUG [StoreOpener-472b3103632c40151a72730c972c9301-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:47,280 INFO [StoreOpener-472b3103632c40151a72730c972c9301-1 {}] regionserver.HStore(327): Store=472b3103632c40151a72730c972c9301/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:57:47,280 INFO [StoreOpener-472b3103632c40151a72730c972c9301-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 472b3103632c40151a72730c972c9301 2024-12-09T23:57:47,281 INFO [StoreOpener-472b3103632c40151a72730c972c9301-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T23:57:47,282 INFO [StoreOpener-472b3103632c40151a72730c972c9301-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 472b3103632c40151a72730c972c9301 columnFamilyName C 2024-12-09T23:57:47,282 DEBUG [StoreOpener-472b3103632c40151a72730c972c9301-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:57:47,282 INFO [StoreOpener-472b3103632c40151a72730c972c9301-1 {}] regionserver.HStore(327): Store=472b3103632c40151a72730c972c9301/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:57:47,282 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:47,282 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for 472b3103632c40151a72730c972c9301 2024-12-09T23:57:47,283 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301 2024-12-09T23:57:47,284 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301 2024-12-09T23:57:47,285 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for 472b3103632c40151a72730c972c9301 2024-12-09T23:57:47,285 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for 472b3103632c40151a72730c972c9301 2024-12-09T23:57:47,285 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T23:57:47,287 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for 472b3103632c40151a72730c972c9301 2024-12-09T23:57:47,289 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T23:57:47,289 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened 472b3103632c40151a72730c972c9301; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70315106, jitterRate=0.0477767288684845}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T23:57:47,289 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 472b3103632c40151a72730c972c9301 2024-12-09T23:57:47,290 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for 472b3103632c40151a72730c972c9301: Running coprocessor pre-open hook at 1733788667273Writing region info on filesystem at 1733788667273Initializing all the Stores at 1733788667274 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788667274Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788667275 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788667275Cleaning up temporary data from old regions at 1733788667285 (+10 ms)Running coprocessor post-open hooks at 1733788667289 (+4 ms)Region opened successfully at 1733788667290 (+1 ms) 2024-12-09T23:57:47,291 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., pid=59, masterSystemTime=1733788667262 2024-12-09T23:57:47,292 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:47,292 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:47,293 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=472b3103632c40151a72730c972c9301, regionState=OPEN, openSeqNum=2, regionLocation=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:47,294 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure 472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 because future has completed 2024-12-09T23:57:47,296 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-09T23:57:47,296 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; OpenRegionProcedure 472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 in 187 msec 2024-12-09T23:57:47,298 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-12-09T23:57:47,298 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=472b3103632c40151a72730c972c9301, ASSIGN in 348 msec 2024-12-09T23:57:47,299 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T23:57:47,299 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733788667299"}]},"ts":"1733788667299"} 2024-12-09T23:57:47,300 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T23:57:47,301 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T23:57:47,303 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1880 sec 2024-12-09T23:57:48,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-09T23:57:48,265 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:57:48,267 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25146acd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:48,268 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:57:48,268 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:57:48,270 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:57:48,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:57:48,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:57:48,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76d4297f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:48,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:57:48,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:57:48,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:48,272 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:57:48,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@370add08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:48,274 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:57:48,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:57:48,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:57:48,276 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48696, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:57:48,277 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T23:57:48,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 2907d75fbb3e,45191,1733788599119 2024-12-09T23:57:48,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@75124bfb 2024-12-09T23:57:48,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T23:57:48,279 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53248, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T23:57:48,280 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@135bd0ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:48,280 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:57:48,280 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:57:48,281 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:57:48,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:57:48,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:57:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a8f2a1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:57:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:57:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:48,283 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53266, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:57:48,283 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7482d7dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:48,284 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4609cead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:48,284 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:57:48,284 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:57:48,285 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:57:48,285 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:57:48,286 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:57:48,286 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@513c327, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:48,286 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:57:48,286 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:57:48,287 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:48,287 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53286, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:57:48,287 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11583f8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:48,288 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a2df097, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:48,288 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:57:48,288 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:57:48,289 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:57:48,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:57:48,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:57:48,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cd3cf66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:48,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:57:48,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:57:48,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:48,290 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53310, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:57:48,290 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4445ae64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:48,291 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2856abc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:48,291 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:57:48,291 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:57:48,291 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:57:48,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:57:48,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:57:48,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c6254c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:48,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:57:48,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:57:48,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:48,292 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53316, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:57:48,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46a8e7ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:48,294 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626de388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:48,294 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:57:48,294 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:57:48,294 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:57:48,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:57:48,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:57:48,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b8c2bbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:48,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:57:48,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:57:48,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:48,295 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:57:48,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f245c73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:48,296 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692cba37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:48,296 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:57:48,296 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:57:48,297 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:57:48,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:57:48,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:57:48,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e625ac7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:48,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:57:48,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:57:48,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:48,298 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53352, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:57:48,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d73772f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:48,299 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:48,299 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:57:48,299 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:57:48,300 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:57:48,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:57:48,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:57:48,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71dc3bce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:48,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:57:48,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:57:48,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:48,301 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53370, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:57:48,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a31a852, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:48,302 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471771d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:48,302 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:57:48,302 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:57:48,302 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:57:48,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:57:48,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:57:48,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65b18ac3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:48,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:57:48,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:57:48,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:48,304 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53390, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:57:48,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e76b18a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:48,305 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40916ac2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:48,305 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:57:48,305 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:57:48,305 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:57:48,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:57:48,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:57:48,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d7289d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:48,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:57:48,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:57:48,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:48,306 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:57:48,307 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3362a79f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:48,307 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12c47ac0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:48,307 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:57:48,307 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:57:48,308 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:57:48,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:57:48,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:57:48,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a0dd573, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:48,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:57:48,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:57:48,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:57:48,309 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53430, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:57:48,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ecc72d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:57:48,310 DEBUG [Thread-978 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:57:48,310 DEBUG [Thread-976 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:57:48,310 DEBUG [Thread-985 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:57:48,310 DEBUG [Thread-983 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:57:48,311 DEBUG [Thread-987 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:57:48,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:57:48,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:57:48,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:57:48,311 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:57:48,311 DEBUG [Thread-970 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:57:48,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:57:48,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:57:48,311 DEBUG [Thread-981 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:57:48,311 DEBUG [Thread-972 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:57:48,311 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:57:48,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:57:48,312 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:57:48,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:57:48,312 DEBUG [Thread-981 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:57:48,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:57:48,312 DEBUG [Thread-981 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:57:48,312 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:57:48,312 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48706, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:57:48,312 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48718, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:57:48,313 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:57:48,313 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48732, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:57:48,313 DEBUG [Thread-989 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:57:48,313 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:57:48,313 DEBUG [Thread-972 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:57:48,313 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48748, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:57:48,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-12-09T23:57:48,314 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48752, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:57:48,314 DEBUG [Thread-972 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:57:48,314 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48762, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:57:48,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-09T23:57:48,315 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:57:48,315 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48772, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:57:48,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:57:48,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2] 2024-12-09T23:57:48,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:57:48,317 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:57:48,317 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:57:48,317 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48784, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:57:48,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2] 2024-12-09T23:57:48,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2] 2024-12-09T23:57:48,318 DEBUG [Thread-974 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:57:48,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2] 2024-12-09T23:57:48,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:57:48,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2] 2024-12-09T23:57:48,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2] 2024-12-09T23:57:48,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:57:48,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2] 2024-12-09T23:57:48,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2] 2024-12-09T23:57:48,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2] 2024-12-09T23:57:48,323 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48800, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:57:48,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2] 2024-12-09T23:57:48,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on 472b3103632c40151a72730c972c9301 2024-12-09T23:57:48,328 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 472b3103632c40151a72730c972c9301 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T23:57:48,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=A 2024-12-09T23:57:48,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:48,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=B 2024-12-09T23:57:48,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:48,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=C 2024-12-09T23:57:48,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:48,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/cfecc2c7d67b432baa19ad68c6318985 is 50, key is test_row_0/A:col10/1733788668328/Put/seqid=0 2024-12-09T23:57:48,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:48,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1733788728356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:48,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:48,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788728357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:48,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:48,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788728357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:48,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:48,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:48,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1733788728359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:48,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:48,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:48,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:48,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:48,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48708 deadline: 1733788728364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:48,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:48,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742028_1204 (size=12001) 2024-12-09T23:57:48,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-09T23:57:48,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-09T23:57:48,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:48,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:57:48,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:48,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:48,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:48,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:48,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:48,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1733788728473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:48,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:48,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1733788728473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:48,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:48,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788728473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:48,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788728473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48708 deadline: 1733788728473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:48,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-09T23:57:48,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:48,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:57:48,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:48,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:48,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:48,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:48,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-09T23:57:48,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:48,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1733788728683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:48,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:48,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:48,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788728683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:48,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:48,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:48,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788728683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:48,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:48,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:48,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:48,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1733788728683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:48,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48708 deadline: 1733788728683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:48,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:48,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:48,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/cfecc2c7d67b432baa19ad68c6318985 2024-12-09T23:57:48,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-09T23:57:48,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:48,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:57:48,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:48,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:48,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:48,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:48,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/928334453a084a708ce7eae376b852b5 is 50, key is test_row_0/B:col10/1733788668328/Put/seqid=0 2024-12-09T23:57:48,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742029_1205 (size=12001) 2024-12-09T23:57:48,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-09T23:57:48,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:48,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:57:48,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:48,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:48,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:48,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:48,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-09T23:57:48,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:48,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:48,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788728993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:48,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1733788728993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:48,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:48,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:48,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:48,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:48,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788728993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:48,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1733788728993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:48,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:48,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:48,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:48,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48708 deadline: 1733788728993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:48,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:48,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:49,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-09T23:57:49,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:49,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:57:49,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:49,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:49,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:49,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:49,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/928334453a084a708ce7eae376b852b5 2024-12-09T23:57:49,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/8175166a8d8d463283eca21fd7241b06 is 50, key is test_row_0/C:col10/1733788668328/Put/seqid=0 2024-12-09T23:57:49,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742030_1206 (size=12001) 2024-12-09T23:57:49,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-09T23:57:49,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:49,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:57:49,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:49,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:49,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:49,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:49,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-09T23:57:49,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:49,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:57:49,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:49,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:49,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:49,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:49,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-09T23:57:49,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:49,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1733788729503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:49,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:49,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:49,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:49,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:49,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788729503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:49,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:49,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:49,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:49,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788729503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:49,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:49,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:49,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:49,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:49,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:49,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1733788729503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:49,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:49,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:49,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:49,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:49,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48708 deadline: 1733788729503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:49,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:49,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:49,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:49,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-09T23:57:49,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:49,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:57:49,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:49,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:49,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:49,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:49,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/8175166a8d8d463283eca21fd7241b06 2024-12-09T23:57:49,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/cfecc2c7d67b432baa19ad68c6318985 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/cfecc2c7d67b432baa19ad68c6318985 2024-12-09T23:57:49,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/cfecc2c7d67b432baa19ad68c6318985, entries=150, sequenceid=13, filesize=11.7 K 2024-12-09T23:57:49,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/928334453a084a708ce7eae376b852b5 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/928334453a084a708ce7eae376b852b5 2024-12-09T23:57:49,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/928334453a084a708ce7eae376b852b5, entries=150, sequenceid=13, filesize=11.7 K 2024-12-09T23:57:49,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/8175166a8d8d463283eca21fd7241b06 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/8175166a8d8d463283eca21fd7241b06 2024-12-09T23:57:49,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/8175166a8d8d463283eca21fd7241b06, entries=150, sequenceid=13, filesize=11.7 K 2024-12-09T23:57:49,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 472b3103632c40151a72730c972c9301 in 1319ms, sequenceid=13, compaction requested=false 2024-12-09T23:57:49,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:57:49,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-09T23:57:49,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:49,694 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 472b3103632c40151a72730c972c9301 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T23:57:49,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=A 2024-12-09T23:57:49,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:49,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=B 2024-12-09T23:57:49,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:49,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=C 2024-12-09T23:57:49,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:49,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/0bb3028debf14d57ae38fc60332c1d77 is 50, key is test_row_0/A:col10/1733788668357/Put/seqid=0 2024-12-09T23:57:49,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742031_1207 (size=12001) 2024-12-09T23:57:50,118 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/0bb3028debf14d57ae38fc60332c1d77 2024-12-09T23:57:50,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/1c17a6c6813d49dfb12f53cd2001f3ff is 50, key is test_row_0/B:col10/1733788668357/Put/seqid=0 2024-12-09T23:57:50,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742032_1208 (size=12001) 2024-12-09T23:57:50,129 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/1c17a6c6813d49dfb12f53cd2001f3ff 2024-12-09T23:57:50,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/fe0e4addc1b54ba4b101e0bf6ab398be is 50, key is test_row_0/C:col10/1733788668357/Put/seqid=0 2024-12-09T23:57:50,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742033_1209 (size=12001) 2024-12-09T23:57:50,158 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/fe0e4addc1b54ba4b101e0bf6ab398be 2024-12-09T23:57:50,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/0bb3028debf14d57ae38fc60332c1d77 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/0bb3028debf14d57ae38fc60332c1d77 2024-12-09T23:57:50,168 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/0bb3028debf14d57ae38fc60332c1d77, entries=150, sequenceid=33, filesize=11.7 K 2024-12-09T23:57:50,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/1c17a6c6813d49dfb12f53cd2001f3ff as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/1c17a6c6813d49dfb12f53cd2001f3ff 2024-12-09T23:57:50,174 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/1c17a6c6813d49dfb12f53cd2001f3ff, entries=150, sequenceid=33, filesize=11.7 K 2024-12-09T23:57:50,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/fe0e4addc1b54ba4b101e0bf6ab398be as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/fe0e4addc1b54ba4b101e0bf6ab398be 2024-12-09T23:57:50,182 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/fe0e4addc1b54ba4b101e0bf6ab398be, entries=150, sequenceid=33, filesize=11.7 K 2024-12-09T23:57:50,183 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 472b3103632c40151a72730c972c9301 in 488ms, sequenceid=33, compaction requested=false 2024-12-09T23:57:50,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:57:50,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:50,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-12-09T23:57:50,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-12-09T23:57:50,187 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-09T23:57:50,187 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8670 sec 2024-12-09T23:57:50,189 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 1.8770 sec 2024-12-09T23:57:50,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-09T23:57:50,462 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:57:50,463 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:57:50,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-12-09T23:57:50,465 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:57:50,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-09T23:57:50,465 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:57:50,466 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:57:50,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on 472b3103632c40151a72730c972c9301 2024-12-09T23:57:50,518 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 472b3103632c40151a72730c972c9301 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T23:57:50,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=A 2024-12-09T23:57:50,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:50,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=B 2024-12-09T23:57:50,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:50,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=C 2024-12-09T23:57:50,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:50,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/d0982414e95c4ea08f7877c23d5863b3 is 50, key is test_row_0/A:col10/1733788670517/Put/seqid=0 2024-12-09T23:57:50,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742034_1210 (size=14341) 2024-12-09T23:57:50,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:50,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788730535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:50,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:50,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:50,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:50,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:50,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788730536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:50,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:50,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:50,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:50,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733788670528, max timestamp=1733788670538Num uniques 859; 2024-12-09T23:57:50,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733788670528, max timestamp=1733788670538Num uniques 859; 2024-12-09T23:57:50,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733788670528, max timestamp=1733788670538Num uniques 859; 2024-12-09T23:57:50,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T23:57:50,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T23:57:50,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T23:57:50,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:50,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1733788730564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:50,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:50,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:50,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:50,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1733788730566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:50,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:50,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48708 deadline: 1733788730565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:50,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:50,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:50,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:50,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:50,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:50,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:50,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:50,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-09T23:57:50,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T23:57:50,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:50,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:57:50,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:50,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:50,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:50,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:50,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:50,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788730642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:50,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:50,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788730642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:50,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:50,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:50,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:50,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:50,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:50,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:50,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:50,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1733788730673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:50,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:50,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:50,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:50,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:50,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48708 deadline: 1733788730673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:50,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:50,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:50,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:50,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:50,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1733788730673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:50,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:50,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:50,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:50,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T23:57:50,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:50,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:57:50,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:50,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:50,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:50,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-09T23:57:50,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:50,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788730852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:50,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:50,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:50,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:50,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:50,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788730852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:50,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:50,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:50,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:50,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:50,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1733788730883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:50,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:50,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:50,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:50,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:50,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48708 deadline: 1733788730883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:50,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:50,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:50,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:50,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:50,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1733788730883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:50,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:50,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:50,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:50,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T23:57:50,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:50,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:57:50,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:50,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:50,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:50,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:50,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/d0982414e95c4ea08f7877c23d5863b3 2024-12-09T23:57:50,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/e6875f0d5221416ea9e398a61457af3c is 50, key is test_row_0/B:col10/1733788670517/Put/seqid=0 2024-12-09T23:57:50,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742035_1211 (size=12001) 2024-12-09T23:57:51,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T23:57:51,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:51,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:57:51,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:51,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:51,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:51,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:51,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-09T23:57:51,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788731163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:51,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:51,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:51,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:51,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:51,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788731163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:51,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:51,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:51,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:51,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:51,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1733788731193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:51,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:51,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48708 deadline: 1733788731193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:51,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:51,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:51,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:51,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:51,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:51,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:51,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:51,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1733788731193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:51,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:51,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:51,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:51,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T23:57:51,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:51,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:57:51,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:51,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:51,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:51,281 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T23:57:51,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/e6875f0d5221416ea9e398a61457af3c 2024-12-09T23:57:51,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/2e96b05db260424f81424df9323d5075 is 50, key is test_row_0/C:col10/1733788670517/Put/seqid=0 2024-12-09T23:57:51,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742036_1212 (size=12001) 2024-12-09T23:57:51,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T23:57:51,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:51,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:57:51,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:51,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:51,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:51,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:51,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T23:57:51,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:51,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:57:51,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:51,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:51,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:51,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-09T23:57:51,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788731672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:51,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788731672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:51,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:51,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:51,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:51,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:51,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:51,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:51,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T23:57:51,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:51,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:57:51,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:51,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:51,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:51,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1733788731703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:51,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:51,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:51,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:51,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48708 deadline: 1733788731703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:51,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:51,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1733788731703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:51,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:51,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:51,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:51,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:51,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:51,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/2e96b05db260424f81424df9323d5075 2024-12-09T23:57:51,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/d0982414e95c4ea08f7877c23d5863b3 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/d0982414e95c4ea08f7877c23d5863b3 2024-12-09T23:57:51,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/d0982414e95c4ea08f7877c23d5863b3, entries=200, sequenceid=44, filesize=14.0 K 2024-12-09T23:57:51,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/e6875f0d5221416ea9e398a61457af3c as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/e6875f0d5221416ea9e398a61457af3c 2024-12-09T23:57:51,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/e6875f0d5221416ea9e398a61457af3c, entries=150, sequenceid=44, filesize=11.7 K 2024-12-09T23:57:51,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/2e96b05db260424f81424df9323d5075 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/2e96b05db260424f81424df9323d5075 2024-12-09T23:57:51,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/2e96b05db260424f81424df9323d5075, entries=150, sequenceid=44, filesize=11.7 K 2024-12-09T23:57:51,768 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 472b3103632c40151a72730c972c9301 in 1250ms, sequenceid=44, compaction requested=true 2024-12-09T23:57:51,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:57:51,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 472b3103632c40151a72730c972c9301:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:57:51,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:51,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 472b3103632c40151a72730c972c9301:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:57:51,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:51,768 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:57:51,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 472b3103632c40151a72730c972c9301:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:57:51,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:57:51,768 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:57:51,769 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:57:51,769 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:57:51,769 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): 472b3103632c40151a72730c972c9301/A is initiating minor compaction (all files) 2024-12-09T23:57:51,769 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): 472b3103632c40151a72730c972c9301/B is initiating minor compaction (all files) 2024-12-09T23:57:51,769 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 472b3103632c40151a72730c972c9301/A in TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:51,769 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 472b3103632c40151a72730c972c9301/B in TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:51,769 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/cfecc2c7d67b432baa19ad68c6318985, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/0bb3028debf14d57ae38fc60332c1d77, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/d0982414e95c4ea08f7877c23d5863b3] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp, totalSize=37.4 K 2024-12-09T23:57:51,769 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/928334453a084a708ce7eae376b852b5, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/1c17a6c6813d49dfb12f53cd2001f3ff, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/e6875f0d5221416ea9e398a61457af3c] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp, totalSize=35.2 K 2024-12-09T23:57:51,770 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfecc2c7d67b432baa19ad68c6318985, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733788668321 2024-12-09T23:57:51,770 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 928334453a084a708ce7eae376b852b5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733788668321 2024-12-09T23:57:51,770 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bb3028debf14d57ae38fc60332c1d77, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733788668355 2024-12-09T23:57:51,770 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c17a6c6813d49dfb12f53cd2001f3ff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733788668355 2024-12-09T23:57:51,770 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0982414e95c4ea08f7877c23d5863b3, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1733788670515 2024-12-09T23:57:51,770 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting e6875f0d5221416ea9e398a61457af3c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1733788670515 2024-12-09T23:57:51,778 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 472b3103632c40151a72730c972c9301#B#compaction#174 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:57:51,779 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/92180db52aaf45c2aae651a0fe32c66f is 50, key is test_row_0/B:col10/1733788670517/Put/seqid=0 2024-12-09T23:57:51,785 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 472b3103632c40151a72730c972c9301#A#compaction#175 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:57:51,786 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/aa6490637aa648198867f224cffea16b is 50, key is test_row_0/A:col10/1733788670517/Put/seqid=0 2024-12-09T23:57:51,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742037_1213 (size=12104) 2024-12-09T23:57:51,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742038_1214 (size=12104) 2024-12-09T23:57:51,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T23:57:51,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:51,840 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 472b3103632c40151a72730c972c9301 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-09T23:57:51,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=A 2024-12-09T23:57:51,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:57:51,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=B 2024-12-09T23:57:51,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:57:51,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=C 2024-12-09T23:57:51,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:57:51,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/6a8f2c92e9ec49be8b9d5000200d89bd is 50, key is test_row_0/A:col10/1733788670565/Put/seqid=0 2024-12-09T23:57:51,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742039_1215 (size=12001) 2024-12-09T23:57:52,199 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/92180db52aaf45c2aae651a0fe32c66f as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/92180db52aaf45c2aae651a0fe32c66f 2024-12-09T23:57:52,203 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 472b3103632c40151a72730c972c9301/B of 472b3103632c40151a72730c972c9301 into 92180db52aaf45c2aae651a0fe32c66f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:52,203 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:57:52,203 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., storeName=472b3103632c40151a72730c972c9301/B, priority=13, startTime=1733788671768; duration=0sec 2024-12-09T23:57:52,203 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:57:52,203 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 472b3103632c40151a72730c972c9301:B 2024-12-09T23:57:52,203 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:57:52,204 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:57:52,205 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): 472b3103632c40151a72730c972c9301/C is initiating minor compaction (all files) 2024-12-09T23:57:52,205 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 472b3103632c40151a72730c972c9301/C in TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:52,205 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/8175166a8d8d463283eca21fd7241b06, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/fe0e4addc1b54ba4b101e0bf6ab398be, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/2e96b05db260424f81424df9323d5075] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp, totalSize=35.2 K 2024-12-09T23:57:52,205 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 8175166a8d8d463283eca21fd7241b06, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733788668321 2024-12-09T23:57:52,205 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting fe0e4addc1b54ba4b101e0bf6ab398be, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733788668355 2024-12-09T23:57:52,206 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e96b05db260424f81424df9323d5075, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1733788670515 2024-12-09T23:57:52,217 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/aa6490637aa648198867f224cffea16b as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/aa6490637aa648198867f224cffea16b 2024-12-09T23:57:52,223 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 472b3103632c40151a72730c972c9301/A of 472b3103632c40151a72730c972c9301 into aa6490637aa648198867f224cffea16b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:52,223 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:57:52,223 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., storeName=472b3103632c40151a72730c972c9301/A, priority=13, startTime=1733788671768; duration=0sec 2024-12-09T23:57:52,223 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:52,223 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 472b3103632c40151a72730c972c9301:A 2024-12-09T23:57:52,225 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 472b3103632c40151a72730c972c9301#C#compaction#177 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:57:52,225 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/cac9b59b13ef41aabe47b5414c8043ac is 50, key is test_row_0/C:col10/1733788670517/Put/seqid=0 2024-12-09T23:57:52,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742040_1216 (size=12104) 2024-12-09T23:57:52,250 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/6a8f2c92e9ec49be8b9d5000200d89bd 2024-12-09T23:57:52,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/836f248d82804c22be0303b473f9620a is 50, key is test_row_0/B:col10/1733788670565/Put/seqid=0 2024-12-09T23:57:52,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742041_1217 (size=12001) 2024-12-09T23:57:52,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-09T23:57:52,653 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/cac9b59b13ef41aabe47b5414c8043ac as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/cac9b59b13ef41aabe47b5414c8043ac 2024-12-09T23:57:52,659 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 472b3103632c40151a72730c972c9301/C of 472b3103632c40151a72730c972c9301 into cac9b59b13ef41aabe47b5414c8043ac(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:52,659 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:57:52,659 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., storeName=472b3103632c40151a72730c972c9301/C, priority=13, startTime=1733788671768; duration=0sec 2024-12-09T23:57:52,659 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:52,659 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 472b3103632c40151a72730c972c9301:C 2024-12-09T23:57:52,661 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/836f248d82804c22be0303b473f9620a 2024-12-09T23:57:52,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/a008c9a79b2c4de185c64ba149138ecb is 50, key is test_row_0/C:col10/1733788670565/Put/seqid=0 2024-12-09T23:57:52,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742042_1218 (size=12001) 2024-12-09T23:57:52,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on 472b3103632c40151a72730c972c9301 2024-12-09T23:57:52,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:57:52,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:52,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788732695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:52,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:52,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:52,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:52,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:52,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788732697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:52,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:52,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:52,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:52,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:52,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48708 deadline: 1733788732712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:52,714 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_1' in 472b3103632c40151a72730c972c9301 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:57:52,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:52,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:52,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:52,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:52,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1733788732723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:52,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:52,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1733788732723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:52,723 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_2' in 472b3103632c40151a72730c972c9301 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:57:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:52,724 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_0' in 472b3103632c40151a72730c972c9301 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:57:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:52,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788732802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:52,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:52,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788732802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:53,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:53,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788733013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:53,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:53,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:53,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788733013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:53,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:53,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:53,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:53,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:53,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:53,073 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/a008c9a79b2c4de185c64ba149138ecb 2024-12-09T23:57:53,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/6a8f2c92e9ec49be8b9d5000200d89bd as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/6a8f2c92e9ec49be8b9d5000200d89bd 2024-12-09T23:57:53,084 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/6a8f2c92e9ec49be8b9d5000200d89bd, entries=150, sequenceid=79, filesize=11.7 K 2024-12-09T23:57:53,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/836f248d82804c22be0303b473f9620a as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/836f248d82804c22be0303b473f9620a 2024-12-09T23:57:53,089 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/836f248d82804c22be0303b473f9620a, entries=150, sequenceid=79, filesize=11.7 K 2024-12-09T23:57:53,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/a008c9a79b2c4de185c64ba149138ecb as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/a008c9a79b2c4de185c64ba149138ecb 2024-12-09T23:57:53,093 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/a008c9a79b2c4de185c64ba149138ecb, entries=150, sequenceid=79, filesize=11.7 K 2024-12-09T23:57:53,094 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 472b3103632c40151a72730c972c9301 in 1255ms, sequenceid=79, compaction requested=false 2024-12-09T23:57:53,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:57:53,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:53,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-12-09T23:57:53,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-12-09T23:57:53,097 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-09T23:57:53,097 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6300 sec 2024-12-09T23:57:53,099 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 2.6350 sec 2024-12-09T23:57:53,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on 472b3103632c40151a72730c972c9301 2024-12-09T23:57:53,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 472b3103632c40151a72730c972c9301 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T23:57:53,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=A 2024-12-09T23:57:53,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:53,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=B 2024-12-09T23:57:53,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:53,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=C 2024-12-09T23:57:53,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:53,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/98ec75e163ac422e927b1bd18adac973 is 50, key is test_row_0/A:col10/1733788673324/Put/seqid=0 2024-12-09T23:57:53,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742043_1219 (size=16681) 2024-12-09T23:57:53,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/98ec75e163ac422e927b1bd18adac973 2024-12-09T23:57:53,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/db83e47692c0419e89287f04e3883fd4 is 50, key is test_row_0/B:col10/1733788673324/Put/seqid=0 2024-12-09T23:57:53,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742044_1220 (size=12001) 2024-12-09T23:57:53,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:53,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788733365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:53,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:53,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:53,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:53,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:53,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788733366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:53,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:53,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:53,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:53,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:53,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:53,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788733473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:53,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788733473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:53,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:53,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:53,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:53,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:53,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:53,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:53,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:53,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788733682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:53,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:53,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:53,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:53,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:53,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788733682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:53,756 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/db83e47692c0419e89287f04e3883fd4 2024-12-09T23:57:53,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/fb25993b8b04491fa0be05d199044190 is 50, key is test_row_0/C:col10/1733788673324/Put/seqid=0 2024-12-09T23:57:53,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742045_1221 (size=12001) 2024-12-09T23:57:53,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/fb25993b8b04491fa0be05d199044190 2024-12-09T23:57:53,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/98ec75e163ac422e927b1bd18adac973 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/98ec75e163ac422e927b1bd18adac973 2024-12-09T23:57:53,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/98ec75e163ac422e927b1bd18adac973, entries=250, sequenceid=95, filesize=16.3 K 2024-12-09T23:57:53,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/db83e47692c0419e89287f04e3883fd4 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/db83e47692c0419e89287f04e3883fd4 2024-12-09T23:57:53,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/db83e47692c0419e89287f04e3883fd4, entries=150, sequenceid=95, filesize=11.7 K 2024-12-09T23:57:53,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/fb25993b8b04491fa0be05d199044190 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/fb25993b8b04491fa0be05d199044190 2024-12-09T23:57:53,797 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/fb25993b8b04491fa0be05d199044190, entries=150, sequenceid=95, filesize=11.7 K 2024-12-09T23:57:53,799 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 472b3103632c40151a72730c972c9301 in 473ms, sequenceid=95, compaction requested=true 2024-12-09T23:57:53,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:57:53,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 472b3103632c40151a72730c972c9301:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:57:53,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:53,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 472b3103632c40151a72730c972c9301:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:57:53,799 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:57:53,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:53,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 472b3103632c40151a72730c972c9301:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:57:53,799 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:57:53,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:57:53,800 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:57:53,800 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:57:53,800 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): 472b3103632c40151a72730c972c9301/B is initiating minor compaction (all files) 2024-12-09T23:57:53,800 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): 472b3103632c40151a72730c972c9301/A is initiating minor compaction (all files) 2024-12-09T23:57:53,800 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 472b3103632c40151a72730c972c9301/B in TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:53,800 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 472b3103632c40151a72730c972c9301/A in TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:53,800 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/92180db52aaf45c2aae651a0fe32c66f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/836f248d82804c22be0303b473f9620a, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/db83e47692c0419e89287f04e3883fd4] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp, totalSize=35.3 K 2024-12-09T23:57:53,800 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/aa6490637aa648198867f224cffea16b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/6a8f2c92e9ec49be8b9d5000200d89bd, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/98ec75e163ac422e927b1bd18adac973] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp, totalSize=39.8 K 2024-12-09T23:57:53,800 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa6490637aa648198867f224cffea16b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1733788670515 2024-12-09T23:57:53,800 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 92180db52aaf45c2aae651a0fe32c66f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1733788670515 2024-12-09T23:57:53,801 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a8f2c92e9ec49be8b9d5000200d89bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733788670563 2024-12-09T23:57:53,801 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 836f248d82804c22be0303b473f9620a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733788670563 2024-12-09T23:57:53,801 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting db83e47692c0419e89287f04e3883fd4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733788672688 2024-12-09T23:57:53,802 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98ec75e163ac422e927b1bd18adac973, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733788672688 2024-12-09T23:57:53,810 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 472b3103632c40151a72730c972c9301#B#compaction#183 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:57:53,810 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/57c01c3d210b4cb9bb572bff8f61cb57 is 50, key is test_row_0/B:col10/1733788673324/Put/seqid=0 2024-12-09T23:57:53,811 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 472b3103632c40151a72730c972c9301#A#compaction#184 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:57:53,812 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/80839f8f625440a980269c155f7f8c14 is 50, key is test_row_0/A:col10/1733788673324/Put/seqid=0 2024-12-09T23:57:53,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742046_1222 (size=12207) 2024-12-09T23:57:53,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742047_1223 (size=12207) 2024-12-09T23:57:53,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on 472b3103632c40151a72730c972c9301 2024-12-09T23:57:53,995 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 472b3103632c40151a72730c972c9301 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T23:57:53,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=A 2024-12-09T23:57:53,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:53,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=B 2024-12-09T23:57:53,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:53,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=C 2024-12-09T23:57:53,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:53,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/8cc9c130d6ff40b4865e36bf9bf87745 is 50, key is test_row_0/A:col10/1733788673994/Put/seqid=0 2024-12-09T23:57:54,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742048_1224 (size=14341) 2024-12-09T23:57:54,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:54,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788734009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:54,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:54,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:54,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:54,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:54,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788734011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:54,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:54,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:54,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:54,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:54,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788734122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:54,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:54,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:54,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:54,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:54,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788734122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:54,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:54,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:54,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:54,231 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/57c01c3d210b4cb9bb572bff8f61cb57 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/57c01c3d210b4cb9bb572bff8f61cb57 2024-12-09T23:57:54,238 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 472b3103632c40151a72730c972c9301/B of 472b3103632c40151a72730c972c9301 into 57c01c3d210b4cb9bb572bff8f61cb57(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:54,239 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:57:54,239 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., storeName=472b3103632c40151a72730c972c9301/B, priority=13, startTime=1733788673799; duration=0sec 2024-12-09T23:57:54,239 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/80839f8f625440a980269c155f7f8c14 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/80839f8f625440a980269c155f7f8c14 2024-12-09T23:57:54,239 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:57:54,239 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 472b3103632c40151a72730c972c9301:B 2024-12-09T23:57:54,239 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:57:54,240 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:57:54,240 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): 472b3103632c40151a72730c972c9301/C is initiating minor compaction (all files) 2024-12-09T23:57:54,241 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 472b3103632c40151a72730c972c9301/C in TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:54,241 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/cac9b59b13ef41aabe47b5414c8043ac, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/a008c9a79b2c4de185c64ba149138ecb, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/fb25993b8b04491fa0be05d199044190] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp, totalSize=35.3 K 2024-12-09T23:57:54,241 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting cac9b59b13ef41aabe47b5414c8043ac, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1733788670515 2024-12-09T23:57:54,242 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting a008c9a79b2c4de185c64ba149138ecb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733788670563 2024-12-09T23:57:54,242 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting fb25993b8b04491fa0be05d199044190, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733788672688 2024-12-09T23:57:54,244 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 472b3103632c40151a72730c972c9301/A of 472b3103632c40151a72730c972c9301 into 80839f8f625440a980269c155f7f8c14(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:54,244 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:57:54,244 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., storeName=472b3103632c40151a72730c972c9301/A, priority=13, startTime=1733788673799; duration=0sec 2024-12-09T23:57:54,244 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:54,244 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 472b3103632c40151a72730c972c9301:A 2024-12-09T23:57:54,250 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 472b3103632c40151a72730c972c9301#C#compaction#186 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:57:54,250 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/2f5e81490f9e4bee92e0714a6f1fb03b is 50, key is test_row_0/C:col10/1733788673324/Put/seqid=0 2024-12-09T23:57:54,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742049_1225 (size=12207) 2024-12-09T23:57:54,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:54,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788734333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:54,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:54,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:54,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:54,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788734333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:54,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:54,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:54,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:54,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:54,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/8cc9c130d6ff40b4865e36bf9bf87745 2024-12-09T23:57:54,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/f1cc363040914da681b1dd241901aa99 is 50, key is test_row_0/B:col10/1733788673994/Put/seqid=0 2024-12-09T23:57:54,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742050_1226 (size=12001) 2024-12-09T23:57:54,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/f1cc363040914da681b1dd241901aa99 2024-12-09T23:57:54,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/24c3f76d344a40089a9b6e7a4431ef0f is 50, key is test_row_0/C:col10/1733788673994/Put/seqid=0 2024-12-09T23:57:54,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742051_1227 (size=12001) 2024-12-09T23:57:54,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-09T23:57:54,643 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:57:54,643 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:57:54,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-09T23:57:54,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-09T23:57:54,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:54,645 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:57:54,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788734643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:54,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:54,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:54,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788734643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:54,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:54,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:54,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:54,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:54,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:54,645 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:57:54,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:57:54,671 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/2f5e81490f9e4bee92e0714a6f1fb03b as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/2f5e81490f9e4bee92e0714a6f1fb03b 2024-12-09T23:57:54,676 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 472b3103632c40151a72730c972c9301/C of 472b3103632c40151a72730c972c9301 into 2f5e81490f9e4bee92e0714a6f1fb03b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:54,677 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:57:54,677 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., storeName=472b3103632c40151a72730c972c9301/C, priority=13, startTime=1733788673799; duration=0sec 2024-12-09T23:57:54,677 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:54,677 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 472b3103632c40151a72730c972c9301:C 2024-12-09T23:57:54,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:54,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48708 deadline: 1733788734733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:54,734 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_1' in 472b3103632c40151a72730c972c9301 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:57:54,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:54,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:54,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:54,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1733788734733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:54,736 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_2' in 472b3103632c40151a72730c972c9301 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:57:54,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:54,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:54,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:54,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1733788734743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:54,744 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_0' in 472b3103632c40151a72730c972c9301 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:57:54,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:54,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:54,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-09T23:57:54,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-09T23:57:54,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:54,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:57:54,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:54,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:54,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:54,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/24c3f76d344a40089a9b6e7a4431ef0f 2024-12-09T23:57:54,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/8cc9c130d6ff40b4865e36bf9bf87745 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/8cc9c130d6ff40b4865e36bf9bf87745 2024-12-09T23:57:54,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/8cc9c130d6ff40b4865e36bf9bf87745, entries=200, sequenceid=115, filesize=14.0 K 2024-12-09T23:57:54,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/f1cc363040914da681b1dd241901aa99 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/f1cc363040914da681b1dd241901aa99 2024-12-09T23:57:54,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/f1cc363040914da681b1dd241901aa99, entries=150, sequenceid=115, filesize=11.7 K 2024-12-09T23:57:54,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/24c3f76d344a40089a9b6e7a4431ef0f as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/24c3f76d344a40089a9b6e7a4431ef0f 2024-12-09T23:57:54,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/24c3f76d344a40089a9b6e7a4431ef0f, entries=150, sequenceid=115, filesize=11.7 K 2024-12-09T23:57:54,855 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 472b3103632c40151a72730c972c9301 in 860ms, sequenceid=115, compaction requested=false 2024-12-09T23:57:54,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:57:54,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-09T23:57:54,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:54,950 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 472b3103632c40151a72730c972c9301 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T23:57:54,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=A 2024-12-09T23:57:54,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:54,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=B 2024-12-09T23:57:54,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:54,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=C 2024-12-09T23:57:54,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:54,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/e4a72d69c2ae425a8074737527eebd84 is 50, key is test_row_0/A:col10/1733788674004/Put/seqid=0 2024-12-09T23:57:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-09T23:57:54,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742052_1228 (size=12001) 2024-12-09T23:57:55,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on 472b3103632c40151a72730c972c9301 2024-12-09T23:57:55,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:57:55,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:55,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788735184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:55,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:55,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:55,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:55,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:55,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788735190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:55,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:55,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:55,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:55,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-09T23:57:55,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:55,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788735292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:55,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:55,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:55,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:55,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:55,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788735302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:55,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:55,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:55,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:55,372 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/e4a72d69c2ae425a8074737527eebd84 2024-12-09T23:57:55,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/7e5ebc4ce174466ab71b55490d5aa764 is 50, key is test_row_0/B:col10/1733788674004/Put/seqid=0 2024-12-09T23:57:55,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742053_1229 (size=12001) 2024-12-09T23:57:55,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:55,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788735503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:55,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:55,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:55,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:55,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:55,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788735512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:55,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:55,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:55,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-09T23:57:55,782 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/7e5ebc4ce174466ab71b55490d5aa764 2024-12-09T23:57:55,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788735813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:55,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:55,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:55,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:55,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788735823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:55,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:55,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:55,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:55,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/02e0a828bf244ba0a110b3091ffdaf5d is 50, key is test_row_0/C:col10/1733788674004/Put/seqid=0 2024-12-09T23:57:55,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742054_1230 (size=12001) 2024-12-09T23:57:56,231 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/02e0a828bf244ba0a110b3091ffdaf5d 2024-12-09T23:57:56,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/e4a72d69c2ae425a8074737527eebd84 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/e4a72d69c2ae425a8074737527eebd84 2024-12-09T23:57:56,239 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/e4a72d69c2ae425a8074737527eebd84, entries=150, sequenceid=130, filesize=11.7 K 2024-12-09T23:57:56,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/7e5ebc4ce174466ab71b55490d5aa764 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/7e5ebc4ce174466ab71b55490d5aa764 2024-12-09T23:57:56,244 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/7e5ebc4ce174466ab71b55490d5aa764, entries=150, sequenceid=130, filesize=11.7 K 2024-12-09T23:57:56,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/02e0a828bf244ba0a110b3091ffdaf5d as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/02e0a828bf244ba0a110b3091ffdaf5d 2024-12-09T23:57:56,249 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/02e0a828bf244ba0a110b3091ffdaf5d, entries=150, sequenceid=130, filesize=11.7 K 2024-12-09T23:57:56,250 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 472b3103632c40151a72730c972c9301 in 1300ms, sequenceid=130, compaction requested=true 2024-12-09T23:57:56,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:57:56,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:56,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-09T23:57:56,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-09T23:57:56,252 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-09T23:57:56,253 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6050 sec 2024-12-09T23:57:56,254 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.6100 sec 2024-12-09T23:57:56,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on 472b3103632c40151a72730c972c9301 2024-12-09T23:57:56,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 472b3103632c40151a72730c972c9301 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-09T23:57:56,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=A 2024-12-09T23:57:56,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:57:56,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=B 2024-12-09T23:57:56,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:57:56,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=C 2024-12-09T23:57:56,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:57:56,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/80bd5dff162d4dd8bbb38fe24e82ea71 is 50, key is test_row_0/A:col10/1733788675189/Put/seqid=0 2024-12-09T23:57:56,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742055_1231 (size=14541) 2024-12-09T23:57:56,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/80bd5dff162d4dd8bbb38fe24e82ea71 2024-12-09T23:57:56,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/3a55c877d75d470abe73a7b3e747147f is 50, key is test_row_0/B:col10/1733788675189/Put/seqid=0 2024-12-09T23:57:56,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742056_1232 (size=12151) 2024-12-09T23:57:56,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/3a55c877d75d470abe73a7b3e747147f 2024-12-09T23:57:56,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:56,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788736345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:56,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:56,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:56,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:56,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/c0b05a09ffd7411f83f839fe51092478 is 50, key is test_row_0/C:col10/1733788675189/Put/seqid=0 2024-12-09T23:57:56,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:56,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788736351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:56,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742057_1233 (size=12151) 2024-12-09T23:57:56,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/c0b05a09ffd7411f83f839fe51092478 2024-12-09T23:57:56,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/80bd5dff162d4dd8bbb38fe24e82ea71 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/80bd5dff162d4dd8bbb38fe24e82ea71 2024-12-09T23:57:56,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/80bd5dff162d4dd8bbb38fe24e82ea71, entries=200, sequenceid=151, filesize=14.2 K 2024-12-09T23:57:56,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/3a55c877d75d470abe73a7b3e747147f as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/3a55c877d75d470abe73a7b3e747147f 2024-12-09T23:57:56,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/3a55c877d75d470abe73a7b3e747147f, entries=150, sequenceid=151, filesize=11.9 K 2024-12-09T23:57:56,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/c0b05a09ffd7411f83f839fe51092478 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/c0b05a09ffd7411f83f839fe51092478 2024-12-09T23:57:56,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/c0b05a09ffd7411f83f839fe51092478, entries=150, sequenceid=151, filesize=11.9 K 2024-12-09T23:57:56,381 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 472b3103632c40151a72730c972c9301 in 56ms, sequenceid=151, compaction requested=true 2024-12-09T23:57:56,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:57:56,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 472b3103632c40151a72730c972c9301:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:57:56,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:56,382 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T23:57:56,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 472b3103632c40151a72730c972c9301:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:57:56,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:56,382 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T23:57:56,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 472b3103632c40151a72730c972c9301:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:57:56,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:57:56,383 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T23:57:56,383 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): 472b3103632c40151a72730c972c9301/B is initiating minor compaction (all files) 2024-12-09T23:57:56,383 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 472b3103632c40151a72730c972c9301/B in TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:56,383 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/57c01c3d210b4cb9bb572bff8f61cb57, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/f1cc363040914da681b1dd241901aa99, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/7e5ebc4ce174466ab71b55490d5aa764, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/3a55c877d75d470abe73a7b3e747147f] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp, totalSize=47.2 K 2024-12-09T23:57:56,383 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T23:57:56,383 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): 472b3103632c40151a72730c972c9301/A is initiating minor compaction (all files) 2024-12-09T23:57:56,383 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 472b3103632c40151a72730c972c9301/A in TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:56,384 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/80839f8f625440a980269c155f7f8c14, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/8cc9c130d6ff40b4865e36bf9bf87745, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/e4a72d69c2ae425a8074737527eebd84, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/80bd5dff162d4dd8bbb38fe24e82ea71] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp, totalSize=51.8 K 2024-12-09T23:57:56,384 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 57c01c3d210b4cb9bb572bff8f61cb57, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733788672688 2024-12-09T23:57:56,384 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80839f8f625440a980269c155f7f8c14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733788672688 2024-12-09T23:57:56,384 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting f1cc363040914da681b1dd241901aa99, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733788673363 2024-12-09T23:57:56,385 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cc9c130d6ff40b4865e36bf9bf87745, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733788673343 2024-12-09T23:57:56,385 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e5ebc4ce174466ab71b55490d5aa764, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733788674004 2024-12-09T23:57:56,385 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4a72d69c2ae425a8074737527eebd84, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733788674004 2024-12-09T23:57:56,385 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a55c877d75d470abe73a7b3e747147f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733788675183 2024-12-09T23:57:56,385 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80bd5dff162d4dd8bbb38fe24e82ea71, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733788675183 2024-12-09T23:57:56,395 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 472b3103632c40151a72730c972c9301#B#compaction#195 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:57:56,395 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 472b3103632c40151a72730c972c9301#A#compaction#196 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:57:56,395 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/2951f28d72a64b3289e77363674dd1e8 is 50, key is test_row_0/B:col10/1733788675189/Put/seqid=0 2024-12-09T23:57:56,395 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/8f2b765900da41099f7d292be536035f is 50, key is test_row_0/A:col10/1733788675189/Put/seqid=0 2024-12-09T23:57:56,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742058_1234 (size=12493) 2024-12-09T23:57:56,409 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/8f2b765900da41099f7d292be536035f as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/8f2b765900da41099f7d292be536035f 2024-12-09T23:57:56,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742059_1235 (size=12493) 2024-12-09T23:57:56,416 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 472b3103632c40151a72730c972c9301/A of 472b3103632c40151a72730c972c9301 into 8f2b765900da41099f7d292be536035f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:56,416 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:57:56,416 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., storeName=472b3103632c40151a72730c972c9301/A, priority=12, startTime=1733788676381; duration=0sec 2024-12-09T23:57:56,416 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:57:56,416 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 472b3103632c40151a72730c972c9301:A 2024-12-09T23:57:56,416 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T23:57:56,418 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T23:57:56,418 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): 472b3103632c40151a72730c972c9301/C is initiating minor compaction (all files) 2024-12-09T23:57:56,418 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 472b3103632c40151a72730c972c9301/C in TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:56,418 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/2f5e81490f9e4bee92e0714a6f1fb03b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/24c3f76d344a40089a9b6e7a4431ef0f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/02e0a828bf244ba0a110b3091ffdaf5d, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/c0b05a09ffd7411f83f839fe51092478] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp, totalSize=47.2 K 2024-12-09T23:57:56,418 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f5e81490f9e4bee92e0714a6f1fb03b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733788672688 2024-12-09T23:57:56,419 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24c3f76d344a40089a9b6e7a4431ef0f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733788673363 2024-12-09T23:57:56,419 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02e0a828bf244ba0a110b3091ffdaf5d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733788674004 2024-12-09T23:57:56,419 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0b05a09ffd7411f83f839fe51092478, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733788675183 2024-12-09T23:57:56,429 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 472b3103632c40151a72730c972c9301#C#compaction#197 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:57:56,430 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/65793e8165e8459f850d84d696984a49 is 50, key is test_row_0/C:col10/1733788675189/Put/seqid=0 2024-12-09T23:57:56,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742060_1236 (size=12493) 2024-12-09T23:57:56,439 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/65793e8165e8459f850d84d696984a49 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/65793e8165e8459f850d84d696984a49 2024-12-09T23:57:56,444 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 472b3103632c40151a72730c972c9301/C of 472b3103632c40151a72730c972c9301 into 65793e8165e8459f850d84d696984a49(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:56,444 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:57:56,444 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., storeName=472b3103632c40151a72730c972c9301/C, priority=12, startTime=1733788676382; duration=0sec 2024-12-09T23:57:56,444 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:56,444 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 472b3103632c40151a72730c972c9301:C 2024-12-09T23:57:56,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on 472b3103632c40151a72730c972c9301 2024-12-09T23:57:56,465 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 472b3103632c40151a72730c972c9301 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T23:57:56,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=A 2024-12-09T23:57:56,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:56,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=B 2024-12-09T23:57:56,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:56,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=C 2024-12-09T23:57:56,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:56,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/bf075875ccae46c0af73bfa7ffc57b9d is 50, key is test_row_0/A:col10/1733788676464/Put/seqid=0 2024-12-09T23:57:56,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742061_1237 (size=16931) 2024-12-09T23:57:56,476 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/bf075875ccae46c0af73bfa7ffc57b9d 2024-12-09T23:57:56,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/5a4ba68dd5fa4adda529da3793b65639 is 50, key is test_row_0/B:col10/1733788676464/Put/seqid=0 2024-12-09T23:57:56,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742062_1238 (size=12151) 2024-12-09T23:57:56,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:56,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788736508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:56,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:56,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:56,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:56,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:56,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788736510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:56,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:56,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:56,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:56,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:56,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788736613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:56,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:56,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:56,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:56,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:56,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788736622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:56,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-09T23:57:56,803 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:57:56,803 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:57:56,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-09T23:57:56,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T23:57:56,805 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:57:56,806 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:57:56,806 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:57:56,818 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/2951f28d72a64b3289e77363674dd1e8 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/2951f28d72a64b3289e77363674dd1e8 2024-12-09T23:57:56,822 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 472b3103632c40151a72730c972c9301/B of 472b3103632c40151a72730c972c9301 into 2951f28d72a64b3289e77363674dd1e8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:56,822 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:57:56,822 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., storeName=472b3103632c40151a72730c972c9301/B, priority=12, startTime=1733788676382; duration=0sec 2024-12-09T23:57:56,822 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:56,822 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 472b3103632c40151a72730c972c9301:B 2024-12-09T23:57:56,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:56,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788736823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:56,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:56,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:56,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:56,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:56,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788736832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:56,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:56,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:56,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:56,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/5a4ba68dd5fa4adda529da3793b65639 2024-12-09T23:57:56,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/ab52f0d20d2144488c49b5d89dc649fc is 50, key is test_row_0/C:col10/1733788676464/Put/seqid=0 2024-12-09T23:57:56,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T23:57:56,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742063_1239 (size=12151) 2024-12-09T23:57:56,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T23:57:56,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:56,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:57:56,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:56,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:56,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:56,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:57,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T23:57:57,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:57,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:57:57,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:57,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:57,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T23:57:57,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788737132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:57,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:57,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:57,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:57,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788737143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:57,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:57,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:57,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:57,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T23:57:57,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:57,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:57:57,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:57,264 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:57,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:57,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/ab52f0d20d2144488c49b5d89dc649fc 2024-12-09T23:57:57,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/bf075875ccae46c0af73bfa7ffc57b9d as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/bf075875ccae46c0af73bfa7ffc57b9d 2024-12-09T23:57:57,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/bf075875ccae46c0af73bfa7ffc57b9d, entries=250, sequenceid=166, filesize=16.5 K 2024-12-09T23:57:57,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/5a4ba68dd5fa4adda529da3793b65639 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/5a4ba68dd5fa4adda529da3793b65639 2024-12-09T23:57:57,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/5a4ba68dd5fa4adda529da3793b65639, entries=150, sequenceid=166, filesize=11.9 K 2024-12-09T23:57:57,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/ab52f0d20d2144488c49b5d89dc649fc as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/ab52f0d20d2144488c49b5d89dc649fc 2024-12-09T23:57:57,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/ab52f0d20d2144488c49b5d89dc649fc, entries=150, sequenceid=166, filesize=11.9 K 2024-12-09T23:57:57,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 472b3103632c40151a72730c972c9301 in 873ms, sequenceid=166, compaction requested=false 2024-12-09T23:57:57,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:57:57,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T23:57:57,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:57,417 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 472b3103632c40151a72730c972c9301 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T23:57:57,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=A 2024-12-09T23:57:57,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:57,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=B 2024-12-09T23:57:57,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:57,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=C 2024-12-09T23:57:57,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:57,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/7b2d8e3699be4404a70c1d604d74c735 is 50, key is test_row_0/A:col10/1733788676498/Put/seqid=0 2024-12-09T23:57:57,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742064_1240 (size=12151) 2024-12-09T23:57:57,426 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/7b2d8e3699be4404a70c1d604d74c735 2024-12-09T23:57:57,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T23:57:57,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/b7b55eecf5144f9da89ddbae14405708 is 50, key is test_row_0/B:col10/1733788676498/Put/seqid=0 2024-12-09T23:57:57,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742065_1241 (size=12151) 2024-12-09T23:57:57,441 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/b7b55eecf5144f9da89ddbae14405708 2024-12-09T23:57:57,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/fd743fb5c46f4666a71c9316cbf9103b is 50, key is test_row_0/C:col10/1733788676498/Put/seqid=0 2024-12-09T23:57:57,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742066_1242 (size=12151) 2024-12-09T23:57:57,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on 472b3103632c40151a72730c972c9301 2024-12-09T23:57:57,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:57:57,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:57,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788737661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:57,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:57,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:57,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:57,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:57,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788737662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:57,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:57,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:57,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:57,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:57,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788737772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:57,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:57,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:57,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788737772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:57,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:57,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:57,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:57,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:57,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:57,854 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/fd743fb5c46f4666a71c9316cbf9103b 2024-12-09T23:57:57,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/7b2d8e3699be4404a70c1d604d74c735 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/7b2d8e3699be4404a70c1d604d74c735 2024-12-09T23:57:57,863 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/7b2d8e3699be4404a70c1d604d74c735, entries=150, sequenceid=186, filesize=11.9 K 2024-12-09T23:57:57,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/b7b55eecf5144f9da89ddbae14405708 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/b7b55eecf5144f9da89ddbae14405708 2024-12-09T23:57:57,868 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/b7b55eecf5144f9da89ddbae14405708, entries=150, sequenceid=186, filesize=11.9 K 2024-12-09T23:57:57,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/fd743fb5c46f4666a71c9316cbf9103b as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/fd743fb5c46f4666a71c9316cbf9103b 2024-12-09T23:57:57,873 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/fd743fb5c46f4666a71c9316cbf9103b, entries=150, sequenceid=186, filesize=11.9 K 2024-12-09T23:57:57,874 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 472b3103632c40151a72730c972c9301 in 456ms, sequenceid=186, compaction requested=true 2024-12-09T23:57:57,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:57:57,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:57,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-09T23:57:57,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-09T23:57:57,877 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-09T23:57:57,877 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0690 sec 2024-12-09T23:57:57,880 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.0760 sec 2024-12-09T23:57:57,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T23:57:57,943 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:57:57,943 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:57:57,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-09T23:57:57,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T23:57:57,945 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:57:57,945 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:57:57,946 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:57:57,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on 472b3103632c40151a72730c972c9301 2024-12-09T23:57:57,987 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 472b3103632c40151a72730c972c9301 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-09T23:57:57,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=A 2024-12-09T23:57:57,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:57,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=B 2024-12-09T23:57:57,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:57,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=C 2024-12-09T23:57:57,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:57,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/be01d367738c4bb89f8cc242137d920c is 50, key is test_row_0/A:col10/1733788677986/Put/seqid=0 2024-12-09T23:57:58,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742067_1243 (size=16931) 2024-12-09T23:57:58,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:58,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788738022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:58,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:58,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:58,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:58,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:58,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788738022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:58,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:58,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:58,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:58,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T23:57:58,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T23:57:58,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:58,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:57:58,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:58,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:58,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:58,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:58,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:58,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:58,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788738132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:58,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788738133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:58,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:58,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:58,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:58,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:58,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:58,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:58,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T23:57:58,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:58,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:57:58,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:58,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:58,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:58,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:58,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T23:57:58,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:58,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788738342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:58,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:58,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:58,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:58,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788738343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:58,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:58,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:58,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:58,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:58,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/be01d367738c4bb89f8cc242137d920c 2024-12-09T23:57:58,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T23:57:58,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:58,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:57:58,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:58,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:58,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:58,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:58,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/46f2cdf369da493bb23821740c722749 is 50, key is test_row_0/B:col10/1733788677986/Put/seqid=0 2024-12-09T23:57:58,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742068_1244 (size=12151) 2024-12-09T23:57:58,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T23:57:58,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:58,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:57:58,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:58,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:58,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:58,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:58,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T23:57:58,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:58,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788738653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:58,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:58,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:58,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:58,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788738653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:58,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:58,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:58,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:58,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:58,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T23:57:58,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:58,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:57:58,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:58,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:58,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:58,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:58,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:58,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48708 deadline: 1733788738743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:58,745 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_1' in 472b3103632c40151a72730c972c9301 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:57:58,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:58,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:58,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:58,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:58,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48800 deadline: 1733788738782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:58,784 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_2' in 472b3103632c40151a72730c972c9301 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:57:58,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:58,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:58,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:58,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:58,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1733788738792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:58,793 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_0' in 472b3103632c40151a72730c972c9301 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:57:58,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:58,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:58,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:58,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/46f2cdf369da493bb23821740c722749 2024-12-09T23:57:58,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/ceaf8b2e5e134762b23431bfdcf0e4af is 50, key is test_row_0/C:col10/1733788677986/Put/seqid=0 2024-12-09T23:57:58,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742069_1245 (size=12151) 2024-12-09T23:57:58,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T23:57:58,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:58,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:57:58,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:58,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:58,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:58,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:59,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T23:57:59,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:59,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:57:59,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:59,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:59,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:59,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:59,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T23:57:59,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:59,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788739162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:59,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:57:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:59,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788739162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:57:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:57:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:57:59,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T23:57:59,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:59,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:57:59,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:59,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:59,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:59,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:57:59,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/ceaf8b2e5e134762b23431bfdcf0e4af 2024-12-09T23:57:59,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/be01d367738c4bb89f8cc242137d920c as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/be01d367738c4bb89f8cc242137d920c 2024-12-09T23:57:59,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/be01d367738c4bb89f8cc242137d920c, entries=250, sequenceid=201, filesize=16.5 K 2024-12-09T23:57:59,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/46f2cdf369da493bb23821740c722749 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/46f2cdf369da493bb23821740c722749 2024-12-09T23:57:59,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/46f2cdf369da493bb23821740c722749, entries=150, sequenceid=201, filesize=11.9 K 2024-12-09T23:57:59,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/ceaf8b2e5e134762b23431bfdcf0e4af as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/ceaf8b2e5e134762b23431bfdcf0e4af 2024-12-09T23:57:59,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/ceaf8b2e5e134762b23431bfdcf0e4af, entries=150, sequenceid=201, filesize=11.9 K 2024-12-09T23:57:59,242 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 472b3103632c40151a72730c972c9301 in 1255ms, sequenceid=201, compaction requested=true 2024-12-09T23:57:59,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:57:59,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 472b3103632c40151a72730c972c9301:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:57:59,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:59,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 472b3103632c40151a72730c972c9301:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:57:59,243 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T23:57:59,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:59,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 472b3103632c40151a72730c972c9301:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:57:59,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:57:59,243 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T23:57:59,244 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T23:57:59,244 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 58506 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T23:57:59,244 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): 472b3103632c40151a72730c972c9301/B is initiating minor compaction (all files) 2024-12-09T23:57:59,244 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): 472b3103632c40151a72730c972c9301/A is initiating minor compaction (all files) 2024-12-09T23:57:59,244 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 472b3103632c40151a72730c972c9301/B in TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:59,244 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 472b3103632c40151a72730c972c9301/A in TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:59,244 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/8f2b765900da41099f7d292be536035f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/bf075875ccae46c0af73bfa7ffc57b9d, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/7b2d8e3699be4404a70c1d604d74c735, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/be01d367738c4bb89f8cc242137d920c] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp, totalSize=57.1 K 2024-12-09T23:57:59,244 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/2951f28d72a64b3289e77363674dd1e8, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/5a4ba68dd5fa4adda529da3793b65639, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/b7b55eecf5144f9da89ddbae14405708, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/46f2cdf369da493bb23821740c722749] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp, totalSize=47.8 K 2024-12-09T23:57:59,244 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 2951f28d72a64b3289e77363674dd1e8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733788675183 2024-12-09T23:57:59,244 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f2b765900da41099f7d292be536035f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733788675183 2024-12-09T23:57:59,244 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a4ba68dd5fa4adda529da3793b65639, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733788676338 2024-12-09T23:57:59,244 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf075875ccae46c0af73bfa7ffc57b9d, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733788676338 2024-12-09T23:57:59,245 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting b7b55eecf5144f9da89ddbae14405708, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733788676498 2024-12-09T23:57:59,245 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b2d8e3699be4404a70c1d604d74c735, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733788676498 2024-12-09T23:57:59,245 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 46f2cdf369da493bb23821740c722749, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733788677657 2024-12-09T23:57:59,245 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting be01d367738c4bb89f8cc242137d920c, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733788677657 2024-12-09T23:57:59,253 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 472b3103632c40151a72730c972c9301#A#compaction#208 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:57:59,254 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/b457e0ba898a4fb08625c78b1b4cbcf8 is 50, key is test_row_0/A:col10/1733788677986/Put/seqid=0 2024-12-09T23:57:59,254 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 472b3103632c40151a72730c972c9301#B#compaction#207 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:57:59,254 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/5d3e4c2609e24214b4961524cd4492c8 is 50, key is test_row_0/B:col10/1733788677986/Put/seqid=0 2024-12-09T23:57:59,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742071_1247 (size=12629) 2024-12-09T23:57:59,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742070_1246 (size=12629) 2024-12-09T23:57:59,272 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/b457e0ba898a4fb08625c78b1b4cbcf8 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/b457e0ba898a4fb08625c78b1b4cbcf8 2024-12-09T23:57:59,272 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/5d3e4c2609e24214b4961524cd4492c8 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/5d3e4c2609e24214b4961524cd4492c8 2024-12-09T23:57:59,277 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 472b3103632c40151a72730c972c9301/A of 472b3103632c40151a72730c972c9301 into b457e0ba898a4fb08625c78b1b4cbcf8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:59,277 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 472b3103632c40151a72730c972c9301/B of 472b3103632c40151a72730c972c9301 into 5d3e4c2609e24214b4961524cd4492c8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:59,277 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:57:59,277 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:57:59,277 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., storeName=472b3103632c40151a72730c972c9301/A, priority=12, startTime=1733788679242; duration=0sec 2024-12-09T23:57:59,277 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., storeName=472b3103632c40151a72730c972c9301/B, priority=12, startTime=1733788679243; duration=0sec 2024-12-09T23:57:59,277 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:57:59,277 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 472b3103632c40151a72730c972c9301:A 2024-12-09T23:57:59,277 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:57:59,277 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 472b3103632c40151a72730c972c9301:B 2024-12-09T23:57:59,277 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T23:57:59,278 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T23:57:59,278 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): 472b3103632c40151a72730c972c9301/C is initiating minor compaction (all files) 2024-12-09T23:57:59,278 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 472b3103632c40151a72730c972c9301/C in TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:59,278 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/65793e8165e8459f850d84d696984a49, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/ab52f0d20d2144488c49b5d89dc649fc, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/fd743fb5c46f4666a71c9316cbf9103b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/ceaf8b2e5e134762b23431bfdcf0e4af] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp, totalSize=47.8 K 2024-12-09T23:57:59,279 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65793e8165e8459f850d84d696984a49, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733788675183 2024-12-09T23:57:59,279 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab52f0d20d2144488c49b5d89dc649fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733788676338 2024-12-09T23:57:59,280 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd743fb5c46f4666a71c9316cbf9103b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733788676498 2024-12-09T23:57:59,280 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting ceaf8b2e5e134762b23431bfdcf0e4af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733788677657 2024-12-09T23:57:59,290 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 472b3103632c40151a72730c972c9301#C#compaction#209 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:57:59,291 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/e8079465af9540538ed501d135632cb4 is 50, key is test_row_0/C:col10/1733788677986/Put/seqid=0 2024-12-09T23:57:59,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742072_1248 (size=12629) 2024-12-09T23:57:59,300 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/e8079465af9540538ed501d135632cb4 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/e8079465af9540538ed501d135632cb4 2024-12-09T23:57:59,304 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 472b3103632c40151a72730c972c9301/C of 472b3103632c40151a72730c972c9301 into e8079465af9540538ed501d135632cb4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:57:59,304 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:57:59,304 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., storeName=472b3103632c40151a72730c972c9301/C, priority=12, startTime=1733788679243; duration=0sec 2024-12-09T23:57:59,305 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:57:59,305 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 472b3103632c40151a72730c972c9301:C 2024-12-09T23:57:59,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T23:57:59,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:57:59,321 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 472b3103632c40151a72730c972c9301 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-09T23:57:59,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=A 2024-12-09T23:57:59,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:59,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=B 2024-12-09T23:57:59,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:59,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=C 2024-12-09T23:57:59,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:57:59,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/71deabdc62af4cdc9fe55c4299cbb51f is 50, key is test_row_0/A:col10/1733788678008/Put/seqid=0 2024-12-09T23:57:59,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742073_1249 (size=12151) 2024-12-09T23:57:59,732 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/71deabdc62af4cdc9fe55c4299cbb51f 2024-12-09T23:57:59,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/7592595a0b1b48febd32b3272540fb80 is 50, key is test_row_0/B:col10/1733788678008/Put/seqid=0 2024-12-09T23:57:59,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742074_1250 (size=12151) 2024-12-09T23:58:00,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T23:58:00,144 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/7592595a0b1b48febd32b3272540fb80 2024-12-09T23:58:00,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/ac238f4817d0467dbd1c619e37f5fa9b is 50, key is test_row_0/C:col10/1733788678008/Put/seqid=0 2024-12-09T23:58:00,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742075_1251 (size=12151) 2024-12-09T23:58:00,157 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/ac238f4817d0467dbd1c619e37f5fa9b 2024-12-09T23:58:00,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/71deabdc62af4cdc9fe55c4299cbb51f as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/71deabdc62af4cdc9fe55c4299cbb51f 2024-12-09T23:58:00,165 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/71deabdc62af4cdc9fe55c4299cbb51f, entries=150, sequenceid=221, filesize=11.9 K 2024-12-09T23:58:00,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/7592595a0b1b48febd32b3272540fb80 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/7592595a0b1b48febd32b3272540fb80 2024-12-09T23:58:00,170 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/7592595a0b1b48febd32b3272540fb80, entries=150, sequenceid=221, filesize=11.9 K 2024-12-09T23:58:00,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/ac238f4817d0467dbd1c619e37f5fa9b as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/ac238f4817d0467dbd1c619e37f5fa9b 2024-12-09T23:58:00,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on 472b3103632c40151a72730c972c9301 2024-12-09T23:58:00,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:58:00,175 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/ac238f4817d0467dbd1c619e37f5fa9b, entries=150, sequenceid=221, filesize=11.9 K 2024-12-09T23:58:00,175 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=13.42 KB/13740 for 472b3103632c40151a72730c972c9301 in 854ms, sequenceid=221, compaction requested=false 2024-12-09T23:58:00,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:58:00,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:00,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-09T23:58:00,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-09T23:58:00,178 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-09T23:58:00,178 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2320 sec 2024-12-09T23:58:00,180 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.2360 sec 2024-12-09T23:58:00,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on 472b3103632c40151a72730c972c9301 2024-12-09T23:58:00,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 472b3103632c40151a72730c972c9301 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T23:58:00,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=A 2024-12-09T23:58:00,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:00,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=B 2024-12-09T23:58:00,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:00,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=C 2024-12-09T23:58:00,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:00,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/c588ebdf210143e782f98aee727a084e is 50, key is test_row_0/A:col10/1733788680178/Put/seqid=0 2024-12-09T23:58:00,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742076_1252 (size=12147) 2024-12-09T23:58:00,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733788680197, max timestamp=1733788680205Num uniques 859; 2024-12-09T23:58:00,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733788680197, max timestamp=1733788680205Num uniques 859; 2024-12-09T23:58:00,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733788680197, max timestamp=1733788680205Num uniques 859; 2024-12-09T23:58:00,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T23:58:00,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T23:58:00,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T23:58:00,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:00,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788740254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:00,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:00,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:00,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:00,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:00,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788740254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:00,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:00,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:00,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:00,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:00,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788740363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:00,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:00,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788740363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:00,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:00,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:00,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:00,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:00,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:00,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:00,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:00,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:00,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788740573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:00,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788740573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:00,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:00,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:00,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:00,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:00,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:00,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:00,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/c588ebdf210143e782f98aee727a084e 2024-12-09T23:58:00,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/f1353beb85da4e7095b1fe8af1780d12 is 50, key is test_row_0/B:col10/1733788680178/Put/seqid=0 2024-12-09T23:58:00,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742077_1253 (size=9757) 2024-12-09T23:58:00,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:00,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788740883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:00,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:00,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:00,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:00,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788740883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:00,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:00,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:00,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:00,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:01,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/f1353beb85da4e7095b1fe8af1780d12 2024-12-09T23:58:01,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/9a00a35fe62e4f5b8ac626013b99a2c2 is 50, key is test_row_0/C:col10/1733788680178/Put/seqid=0 2024-12-09T23:58:01,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742078_1254 (size=9757) 2024-12-09T23:58:01,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:01,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788741393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:01,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:01,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:01,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:01,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788741393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:01,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:01,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:01,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:01,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:01,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/9a00a35fe62e4f5b8ac626013b99a2c2 2024-12-09T23:58:01,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/c588ebdf210143e782f98aee727a084e as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/c588ebdf210143e782f98aee727a084e 2024-12-09T23:58:01,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/c588ebdf210143e782f98aee727a084e, entries=150, sequenceid=231, filesize=11.9 K 2024-12-09T23:58:01,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/f1353beb85da4e7095b1fe8af1780d12 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/f1353beb85da4e7095b1fe8af1780d12 2024-12-09T23:58:01,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/f1353beb85da4e7095b1fe8af1780d12, entries=100, sequenceid=231, filesize=9.5 K 2024-12-09T23:58:01,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/9a00a35fe62e4f5b8ac626013b99a2c2 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/9a00a35fe62e4f5b8ac626013b99a2c2 2024-12-09T23:58:01,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/9a00a35fe62e4f5b8ac626013b99a2c2, entries=100, sequenceid=231, filesize=9.5 K 2024-12-09T23:58:01,439 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 472b3103632c40151a72730c972c9301 in 1258ms, sequenceid=231, compaction requested=true 2024-12-09T23:58:01,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:58:01,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 472b3103632c40151a72730c972c9301:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:58:01,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:01,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 472b3103632c40151a72730c972c9301:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:58:01,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:01,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 472b3103632c40151a72730c972c9301:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:58:01,439 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:58:01,439 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:58:01,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:58:01,440 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36927 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:58:01,440 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34537 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:58:01,440 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): 472b3103632c40151a72730c972c9301/B is initiating minor compaction (all files) 2024-12-09T23:58:01,440 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): 472b3103632c40151a72730c972c9301/A is initiating minor compaction (all files) 2024-12-09T23:58:01,440 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 472b3103632c40151a72730c972c9301/B in TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:01,440 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 472b3103632c40151a72730c972c9301/A in TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:01,440 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/b457e0ba898a4fb08625c78b1b4cbcf8, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/71deabdc62af4cdc9fe55c4299cbb51f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/c588ebdf210143e782f98aee727a084e] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp, totalSize=36.1 K 2024-12-09T23:58:01,440 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/5d3e4c2609e24214b4961524cd4492c8, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/7592595a0b1b48febd32b3272540fb80, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/f1353beb85da4e7095b1fe8af1780d12] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp, totalSize=33.7 K 2024-12-09T23:58:01,440 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting b457e0ba898a4fb08625c78b1b4cbcf8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733788677657 2024-12-09T23:58:01,440 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d3e4c2609e24214b4961524cd4492c8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733788677657 2024-12-09T23:58:01,441 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 7592595a0b1b48febd32b3272540fb80, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733788678008 2024-12-09T23:58:01,441 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71deabdc62af4cdc9fe55c4299cbb51f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733788678008 2024-12-09T23:58:01,441 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting c588ebdf210143e782f98aee727a084e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1733788680176 2024-12-09T23:58:01,441 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting f1353beb85da4e7095b1fe8af1780d12, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1733788680178 2024-12-09T23:58:01,448 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 472b3103632c40151a72730c972c9301#A#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:01,448 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 472b3103632c40151a72730c972c9301#B#compaction#217 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:01,448 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/476d2b1b10d44a278ac944bc7c9066b4 is 50, key is test_row_0/A:col10/1733788680178/Put/seqid=0 2024-12-09T23:58:01,449 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/69265fec8d1f40779b0b5a7b6484a977 is 50, key is test_row_0/B:col10/1733788680178/Put/seqid=0 2024-12-09T23:58:01,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742079_1255 (size=12731) 2024-12-09T23:58:01,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742080_1256 (size=12731) 2024-12-09T23:58:01,469 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/476d2b1b10d44a278ac944bc7c9066b4 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/476d2b1b10d44a278ac944bc7c9066b4 2024-12-09T23:58:01,469 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/69265fec8d1f40779b0b5a7b6484a977 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/69265fec8d1f40779b0b5a7b6484a977 2024-12-09T23:58:01,473 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 472b3103632c40151a72730c972c9301/A of 472b3103632c40151a72730c972c9301 into 476d2b1b10d44a278ac944bc7c9066b4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:01,473 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:58:01,473 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., storeName=472b3103632c40151a72730c972c9301/A, priority=13, startTime=1733788681439; duration=0sec 2024-12-09T23:58:01,473 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:58:01,473 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 472b3103632c40151a72730c972c9301:A 2024-12-09T23:58:01,473 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:58:01,474 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 472b3103632c40151a72730c972c9301/B of 472b3103632c40151a72730c972c9301 into 69265fec8d1f40779b0b5a7b6484a977(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:01,474 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:58:01,474 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., storeName=472b3103632c40151a72730c972c9301/B, priority=13, startTime=1733788681439; duration=0sec 2024-12-09T23:58:01,474 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:01,474 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 472b3103632c40151a72730c972c9301:B 2024-12-09T23:58:01,475 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34537 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:58:01,475 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): 472b3103632c40151a72730c972c9301/C is initiating minor compaction (all files) 2024-12-09T23:58:01,475 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 472b3103632c40151a72730c972c9301/C in TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:01,475 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/e8079465af9540538ed501d135632cb4, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/ac238f4817d0467dbd1c619e37f5fa9b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/9a00a35fe62e4f5b8ac626013b99a2c2] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp, totalSize=33.7 K 2024-12-09T23:58:01,476 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8079465af9540538ed501d135632cb4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733788677657 2024-12-09T23:58:01,476 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac238f4817d0467dbd1c619e37f5fa9b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733788678008 2024-12-09T23:58:01,476 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a00a35fe62e4f5b8ac626013b99a2c2, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1733788680178 2024-12-09T23:58:01,482 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 472b3103632c40151a72730c972c9301#C#compaction#218 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:01,483 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/6da666b57dbc4b959acf200d6a81558e is 50, key is test_row_0/C:col10/1733788680178/Put/seqid=0 2024-12-09T23:58:01,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742081_1257 (size=12731) 2024-12-09T23:58:01,898 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/6da666b57dbc4b959acf200d6a81558e as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/6da666b57dbc4b959acf200d6a81558e 2024-12-09T23:58:01,903 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 472b3103632c40151a72730c972c9301/C of 472b3103632c40151a72730c972c9301 into 6da666b57dbc4b959acf200d6a81558e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:01,903 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:58:01,903 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., storeName=472b3103632c40151a72730c972c9301/C, priority=13, startTime=1733788681439; duration=0sec 2024-12-09T23:58:01,903 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:01,903 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 472b3103632c40151a72730c972c9301:C 2024-12-09T23:58:02,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T23:58:02,103 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:58:02,104 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:58:02,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-09T23:58:02,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T23:58:02,105 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:58:02,106 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:58:02,106 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:58:02,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T23:58:02,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-09T23:58:02,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:02,258 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 472b3103632c40151a72730c972c9301 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-09T23:58:02,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=A 2024-12-09T23:58:02,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:58:02,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=B 2024-12-09T23:58:02,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:58:02,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=C 2024-12-09T23:58:02,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:58:02,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/be2ccff95b7349d2a04b5efca4746d63 is 50, key is test_row_0/A:col10/1733788680252/Put/seqid=0 2024-12-09T23:58:02,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742082_1258 (size=12301) 2024-12-09T23:58:02,271 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/be2ccff95b7349d2a04b5efca4746d63 2024-12-09T23:58:02,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/4f3958c6352c4866b41c1bb7fdc236e3 is 50, key is test_row_0/B:col10/1733788680252/Put/seqid=0 2024-12-09T23:58:02,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742083_1259 (size=12301) 2024-12-09T23:58:02,287 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/4f3958c6352c4866b41c1bb7fdc236e3 2024-12-09T23:58:02,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/b1cf2075d8314d25a790c5034c9aa17c is 50, key is test_row_0/C:col10/1733788680252/Put/seqid=0 2024-12-09T23:58:02,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742084_1260 (size=12301) 2024-12-09T23:58:02,323 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/b1cf2075d8314d25a790c5034c9aa17c 2024-12-09T23:58:02,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/be2ccff95b7349d2a04b5efca4746d63 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/be2ccff95b7349d2a04b5efca4746d63 2024-12-09T23:58:02,332 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/be2ccff95b7349d2a04b5efca4746d63, entries=150, sequenceid=270, filesize=12.0 K 2024-12-09T23:58:02,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/4f3958c6352c4866b41c1bb7fdc236e3 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/4f3958c6352c4866b41c1bb7fdc236e3 2024-12-09T23:58:02,337 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/4f3958c6352c4866b41c1bb7fdc236e3, entries=150, sequenceid=270, filesize=12.0 K 2024-12-09T23:58:02,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/b1cf2075d8314d25a790c5034c9aa17c as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/b1cf2075d8314d25a790c5034c9aa17c 2024-12-09T23:58:02,347 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/b1cf2075d8314d25a790c5034c9aa17c, entries=150, sequenceid=270, filesize=12.0 K 2024-12-09T23:58:02,348 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=0 B/0 for 472b3103632c40151a72730c972c9301 in 90ms, sequenceid=270, compaction requested=false 2024-12-09T23:58:02,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:58:02,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:02,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-09T23:58:02,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-09T23:58:02,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-09T23:58:02,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 243 msec 2024-12-09T23:58:02,353 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 248 msec 2024-12-09T23:58:02,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on 472b3103632c40151a72730c972c9301 2024-12-09T23:58:02,415 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 472b3103632c40151a72730c972c9301 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T23:58:02,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=A 2024-12-09T23:58:02,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:02,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=B 2024-12-09T23:58:02,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:02,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=C 2024-12-09T23:58:02,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:02,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/7c5b1c55d7c5446a94d414adfa9d8e1f is 50, key is test_row_0/A:col10/1733788682414/Put/seqid=0 2024-12-09T23:58:02,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T23:58:02,423 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:58:02,423 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:58:02,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742085_1261 (size=12301) 2024-12-09T23:58:02,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/7c5b1c55d7c5446a94d414adfa9d8e1f 2024-12-09T23:58:02,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-09T23:58:02,426 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:58:02,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T23:58:02,426 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:58:02,427 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:58:02,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/cad4ba9e52404f9b90317bd150444851 is 50, key is test_row_0/B:col10/1733788682414/Put/seqid=0 2024-12-09T23:58:02,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742086_1262 (size=12301) 2024-12-09T23:58:02,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733788682451, max timestamp=1733788682462Num uniques 859; 2024-12-09T23:58:02,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T23:58:02,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733788682451, max timestamp=1733788682462Num uniques 859; 2024-12-09T23:58:02,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T23:58:02,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733788682451, max timestamp=1733788682462Num uniques 859; 2024-12-09T23:58:02,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T23:58:02,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:02,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788742517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:02,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:02,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:02,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:02,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788742517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:02,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:02,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:02,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:02,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:02,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T23:58:02,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-09T23:58:02,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:02,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:58:02,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:02,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:02,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:02,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:02,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:02,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788742632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:02,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:02,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788742632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:02,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:02,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:02,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:02,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:02,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:02,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:02,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-09T23:58:02,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:02,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:58:02,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:02,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:02,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:02,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:02,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T23:58:02,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/cad4ba9e52404f9b90317bd150444851 2024-12-09T23:58:02,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/c6ee59f13a184aefa865010e3ed5ed2c is 50, key is test_row_0/C:col10/1733788682414/Put/seqid=0 2024-12-09T23:58:02,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:02,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788742843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:02,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:02,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:02,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:02,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:02,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788742845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:02,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:02,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:02,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:02,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742087_1263 (size=12301) 2024-12-09T23:58:02,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/c6ee59f13a184aefa865010e3ed5ed2c 2024-12-09T23:58:02,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/7c5b1c55d7c5446a94d414adfa9d8e1f as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/7c5b1c55d7c5446a94d414adfa9d8e1f 2024-12-09T23:58:02,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/7c5b1c55d7c5446a94d414adfa9d8e1f, entries=150, sequenceid=280, filesize=12.0 K 2024-12-09T23:58:02,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/cad4ba9e52404f9b90317bd150444851 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/cad4ba9e52404f9b90317bd150444851 2024-12-09T23:58:02,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/cad4ba9e52404f9b90317bd150444851, entries=150, sequenceid=280, filesize=12.0 K 2024-12-09T23:58:02,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/c6ee59f13a184aefa865010e3ed5ed2c as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/c6ee59f13a184aefa865010e3ed5ed2c 2024-12-09T23:58:02,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/c6ee59f13a184aefa865010e3ed5ed2c, entries=150, sequenceid=280, filesize=12.0 K 2024-12-09T23:58:02,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 472b3103632c40151a72730c972c9301 in 453ms, sequenceid=280, compaction requested=true 2024-12-09T23:58:02,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:58:02,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 472b3103632c40151a72730c972c9301:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:58:02,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:02,868 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:58:02,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 472b3103632c40151a72730c972c9301:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:58:02,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:02,869 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:58:02,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 472b3103632c40151a72730c972c9301:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:58:02,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:58:02,869 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:58:02,869 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:58:02,870 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): 472b3103632c40151a72730c972c9301/A is initiating minor compaction (all files) 2024-12-09T23:58:02,870 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): 472b3103632c40151a72730c972c9301/B is initiating minor compaction (all files) 2024-12-09T23:58:02,870 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 472b3103632c40151a72730c972c9301/A in TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:02,870 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 472b3103632c40151a72730c972c9301/B in TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:02,870 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/476d2b1b10d44a278ac944bc7c9066b4, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/be2ccff95b7349d2a04b5efca4746d63, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/7c5b1c55d7c5446a94d414adfa9d8e1f] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp, totalSize=36.5 K 2024-12-09T23:58:02,870 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/69265fec8d1f40779b0b5a7b6484a977, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/4f3958c6352c4866b41c1bb7fdc236e3, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/cad4ba9e52404f9b90317bd150444851] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp, totalSize=36.5 K 2024-12-09T23:58:02,870 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 476d2b1b10d44a278ac944bc7c9066b4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1733788678020 2024-12-09T23:58:02,870 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 69265fec8d1f40779b0b5a7b6484a977, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1733788678020 2024-12-09T23:58:02,870 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting be2ccff95b7349d2a04b5efca4746d63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1733788680227 2024-12-09T23:58:02,870 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f3958c6352c4866b41c1bb7fdc236e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1733788680227 2024-12-09T23:58:02,870 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c5b1c55d7c5446a94d414adfa9d8e1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733788682405 2024-12-09T23:58:02,871 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting cad4ba9e52404f9b90317bd150444851, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733788682405 2024-12-09T23:58:02,878 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 472b3103632c40151a72730c972c9301#A#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:02,878 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 472b3103632c40151a72730c972c9301#B#compaction#226 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:02,878 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/a17fb33618a645f799c296a124260e8f is 50, key is test_row_0/A:col10/1733788682414/Put/seqid=0 2024-12-09T23:58:02,878 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/3a6b70d86d3d488a83a032cfff5b402a is 50, key is test_row_0/B:col10/1733788682414/Put/seqid=0 2024-12-09T23:58:02,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742088_1264 (size=12983) 2024-12-09T23:58:02,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742089_1265 (size=12983) 2024-12-09T23:58:02,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-09T23:58:02,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:02,886 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 472b3103632c40151a72730c972c9301 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-09T23:58:02,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=A 2024-12-09T23:58:02,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:58:02,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=B 2024-12-09T23:58:02,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:58:02,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=C 2024-12-09T23:58:02,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:58:02,889 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/3a6b70d86d3d488a83a032cfff5b402a as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/3a6b70d86d3d488a83a032cfff5b402a 2024-12-09T23:58:02,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/c0dfca9e6f7443c0a76402125fb2aedd is 50, key is test_row_0/A:col10/1733788682504/Put/seqid=0 2024-12-09T23:58:02,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742090_1266 (size=12301) 2024-12-09T23:58:02,896 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 472b3103632c40151a72730c972c9301/B of 472b3103632c40151a72730c972c9301 into 3a6b70d86d3d488a83a032cfff5b402a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:02,896 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:58:02,896 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., storeName=472b3103632c40151a72730c972c9301/B, priority=13, startTime=1733788682869; duration=0sec 2024-12-09T23:58:02,896 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:58:02,896 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 472b3103632c40151a72730c972c9301:B 2024-12-09T23:58:02,897 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:58:02,898 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:58:02,898 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): 472b3103632c40151a72730c972c9301/C is initiating minor compaction (all files) 2024-12-09T23:58:02,898 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 472b3103632c40151a72730c972c9301/C in TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:02,898 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/6da666b57dbc4b959acf200d6a81558e, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/b1cf2075d8314d25a790c5034c9aa17c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/c6ee59f13a184aefa865010e3ed5ed2c] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp, totalSize=36.5 K 2024-12-09T23:58:02,899 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 6da666b57dbc4b959acf200d6a81558e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1733788678020 2024-12-09T23:58:02,899 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting b1cf2075d8314d25a790c5034c9aa17c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1733788680227 2024-12-09T23:58:02,899 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting c6ee59f13a184aefa865010e3ed5ed2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733788682405 2024-12-09T23:58:02,907 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 472b3103632c40151a72730c972c9301#C#compaction#228 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:02,908 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/ed4bc1ec726c45b291d6157aef868697 is 50, key is test_row_0/C:col10/1733788682414/Put/seqid=0 2024-12-09T23:58:02,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742091_1267 (size=12983) 2024-12-09T23:58:03,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T23:58:03,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on 472b3103632c40151a72730c972c9301 2024-12-09T23:58:03,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:58:03,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788743163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:03,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:03,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:03,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:03,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:03,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788743167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:03,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:03,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:03,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:03,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:03,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788743273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:03,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:03,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:03,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:03,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788743273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:03,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:03,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:03,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:03,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:03,291 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/a17fb33618a645f799c296a124260e8f as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/a17fb33618a645f799c296a124260e8f 2024-12-09T23:58:03,295 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 472b3103632c40151a72730c972c9301/A of 472b3103632c40151a72730c972c9301 into a17fb33618a645f799c296a124260e8f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:03,295 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/c0dfca9e6f7443c0a76402125fb2aedd 2024-12-09T23:58:03,295 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:58:03,295 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., storeName=472b3103632c40151a72730c972c9301/A, priority=13, startTime=1733788682868; duration=0sec 2024-12-09T23:58:03,295 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:03,295 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 472b3103632c40151a72730c972c9301:A 2024-12-09T23:58:03,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/ca4f7107693440fc9e822290af0a797d is 50, key is test_row_0/B:col10/1733788682504/Put/seqid=0 2024-12-09T23:58:03,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742092_1268 (size=12301) 2024-12-09T23:58:03,316 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/ed4bc1ec726c45b291d6157aef868697 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/ed4bc1ec726c45b291d6157aef868697 2024-12-09T23:58:03,321 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 472b3103632c40151a72730c972c9301/C of 472b3103632c40151a72730c972c9301 into ed4bc1ec726c45b291d6157aef868697(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:03,321 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:58:03,321 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., storeName=472b3103632c40151a72730c972c9301/C, priority=13, startTime=1733788682869; duration=0sec 2024-12-09T23:58:03,321 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:03,321 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 472b3103632c40151a72730c972c9301:C 2024-12-09T23:58:03,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:03,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788743483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:03,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:03,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:03,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788743483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:03,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:03,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:03,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:03,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:03,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:03,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T23:58:03,704 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/ca4f7107693440fc9e822290af0a797d 2024-12-09T23:58:03,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/76681ffa14f64e1ab426423f953997f2 is 50, key is test_row_0/C:col10/1733788682504/Put/seqid=0 2024-12-09T23:58:03,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742093_1269 (size=12301) 2024-12-09T23:58:03,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:03,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788743793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:03,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:03,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:03,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:03,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:03,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788743793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:03,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:03,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:03,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:04,115 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/76681ffa14f64e1ab426423f953997f2 2024-12-09T23:58:04,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/c0dfca9e6f7443c0a76402125fb2aedd as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/c0dfca9e6f7443c0a76402125fb2aedd 2024-12-09T23:58:04,123 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/c0dfca9e6f7443c0a76402125fb2aedd, entries=150, sequenceid=317, filesize=12.0 K 2024-12-09T23:58:04,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/ca4f7107693440fc9e822290af0a797d as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/ca4f7107693440fc9e822290af0a797d 2024-12-09T23:58:04,127 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/ca4f7107693440fc9e822290af0a797d, entries=150, sequenceid=317, filesize=12.0 K 2024-12-09T23:58:04,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/76681ffa14f64e1ab426423f953997f2 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/76681ffa14f64e1ab426423f953997f2 2024-12-09T23:58:04,132 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/76681ffa14f64e1ab426423f953997f2, entries=150, sequenceid=317, filesize=12.0 K 2024-12-09T23:58:04,133 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 472b3103632c40151a72730c972c9301 in 1247ms, sequenceid=317, compaction requested=false 2024-12-09T23:58:04,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:58:04,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:04,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-09T23:58:04,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-09T23:58:04,135 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-09T23:58:04,136 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7080 sec 2024-12-09T23:58:04,137 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.7130 sec 2024-12-09T23:58:04,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on 472b3103632c40151a72730c972c9301 2024-12-09T23:58:04,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 472b3103632c40151a72730c972c9301 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T23:58:04,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=A 2024-12-09T23:58:04,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:04,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=B 2024-12-09T23:58:04,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:04,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=C 2024-12-09T23:58:04,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:04,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/95e2142f7a054299a0ec8ce3090debe2 is 50, key is test_row_0/A:col10/1733788683165/Put/seqid=0 2024-12-09T23:58:04,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742094_1270 (size=14741) 2024-12-09T23:58:04,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:04,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788744332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:04,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:04,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:04,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:04,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:04,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788744334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:04,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:04,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:04,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:04,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:04,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788744443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:04,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:04,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:04,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788744443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:04,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:04,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:04,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:04,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:04,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:04,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T23:58:04,572 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:58:04,573 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:58:04,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-09T23:58:04,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T23:58:04,574 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:58:04,575 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:58:04,575 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:58:04,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:04,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788744653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:04,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:04,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:04,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:04,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788744653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:04,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:04,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:04,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:04,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:04,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T23:58:04,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/95e2142f7a054299a0ec8ce3090debe2 2024-12-09T23:58:04,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/944ac4c34ea74e9b9993023006b067d4 is 50, key is test_row_0/B:col10/1733788683165/Put/seqid=0 2024-12-09T23:58:04,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-09T23:58:04,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:04,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:58:04,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:04,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:04,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:04,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:04,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742095_1271 (size=12301) 2024-12-09T23:58:04,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-09T23:58:04,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:04,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:58:04,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:04,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:04,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:04,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:04,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T23:58:04,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:04,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788744962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:04,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:04,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:04,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788744962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:04,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:04,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:04,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:04,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:04,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:05,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-09T23:58:05,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:05,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:58:05,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:05,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:05,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:05,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:05,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/944ac4c34ea74e9b9993023006b067d4 2024-12-09T23:58:05,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/0290395aa7714988979eaed5603832dd is 50, key is test_row_0/C:col10/1733788683165/Put/seqid=0 2024-12-09T23:58:05,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742096_1272 (size=12301) 2024-12-09T23:58:05,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-09T23:58:05,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:05,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:58:05,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:05,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:05,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:05,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:05,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T23:58:05,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-09T23:58:05,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:05,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:58:05,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:05,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:05,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:05,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:05,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:05,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788745472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:05,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:05,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:05,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:05,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:05,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788745472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:05,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:05,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:05,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:05,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-09T23:58:05,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:05,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:58:05,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:05,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:05,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:05,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:05,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/0290395aa7714988979eaed5603832dd 2024-12-09T23:58:05,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/95e2142f7a054299a0ec8ce3090debe2 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/95e2142f7a054299a0ec8ce3090debe2 2024-12-09T23:58:05,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/95e2142f7a054299a0ec8ce3090debe2, entries=200, sequenceid=332, filesize=14.4 K 2024-12-09T23:58:05,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/944ac4c34ea74e9b9993023006b067d4 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/944ac4c34ea74e9b9993023006b067d4 2024-12-09T23:58:05,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/944ac4c34ea74e9b9993023006b067d4, entries=150, sequenceid=332, filesize=12.0 K 2024-12-09T23:58:05,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/0290395aa7714988979eaed5603832dd as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/0290395aa7714988979eaed5603832dd 2024-12-09T23:58:05,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/0290395aa7714988979eaed5603832dd, entries=150, sequenceid=332, filesize=12.0 K 2024-12-09T23:58:05,560 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 472b3103632c40151a72730c972c9301 in 1255ms, sequenceid=332, compaction requested=true 2024-12-09T23:58:05,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:58:05,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 472b3103632c40151a72730c972c9301:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:58:05,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:05,560 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:58:05,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 472b3103632c40151a72730c972c9301:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:58:05,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:05,560 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:58:05,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 472b3103632c40151a72730c972c9301:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:58:05,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:58:05,561 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:58:05,561 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:58:05,561 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): 472b3103632c40151a72730c972c9301/A is initiating minor compaction (all files) 2024-12-09T23:58:05,561 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): 472b3103632c40151a72730c972c9301/B is initiating minor compaction (all files) 2024-12-09T23:58:05,561 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 472b3103632c40151a72730c972c9301/A in TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:05,561 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 472b3103632c40151a72730c972c9301/B in TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:05,561 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/a17fb33618a645f799c296a124260e8f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/c0dfca9e6f7443c0a76402125fb2aedd, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/95e2142f7a054299a0ec8ce3090debe2] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp, totalSize=39.1 K 2024-12-09T23:58:05,561 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/3a6b70d86d3d488a83a032cfff5b402a, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/ca4f7107693440fc9e822290af0a797d, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/944ac4c34ea74e9b9993023006b067d4] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp, totalSize=36.7 K 2024-12-09T23:58:05,562 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a6b70d86d3d488a83a032cfff5b402a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733788682405 2024-12-09T23:58:05,562 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting a17fb33618a645f799c296a124260e8f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733788682405 2024-12-09T23:58:05,562 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting ca4f7107693440fc9e822290af0a797d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733788682504 2024-12-09T23:58:05,562 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0dfca9e6f7443c0a76402125fb2aedd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733788682504 2024-12-09T23:58:05,562 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 944ac4c34ea74e9b9993023006b067d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733788683162 2024-12-09T23:58:05,562 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95e2142f7a054299a0ec8ce3090debe2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733788683162 2024-12-09T23:58:05,570 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 472b3103632c40151a72730c972c9301#A#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:05,570 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/90a715e6ccba405ab8722e22e7ae14ff is 50, key is test_row_0/A:col10/1733788683165/Put/seqid=0 2024-12-09T23:58:05,575 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 472b3103632c40151a72730c972c9301#B#compaction#235 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:05,575 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/7fcfab5c201a4125bf9366088c77a00d is 50, key is test_row_0/B:col10/1733788683165/Put/seqid=0 2024-12-09T23:58:05,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742097_1273 (size=13085) 2024-12-09T23:58:05,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742098_1274 (size=13085) 2024-12-09T23:58:05,589 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/7fcfab5c201a4125bf9366088c77a00d as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/7fcfab5c201a4125bf9366088c77a00d 2024-12-09T23:58:05,594 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 472b3103632c40151a72730c972c9301/B of 472b3103632c40151a72730c972c9301 into 7fcfab5c201a4125bf9366088c77a00d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:05,594 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:58:05,594 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., storeName=472b3103632c40151a72730c972c9301/B, priority=13, startTime=1733788685560; duration=0sec 2024-12-09T23:58:05,594 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:58:05,595 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 472b3103632c40151a72730c972c9301:B 2024-12-09T23:58:05,595 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:58:05,596 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:58:05,596 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): 472b3103632c40151a72730c972c9301/C is initiating minor compaction (all files) 2024-12-09T23:58:05,596 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 472b3103632c40151a72730c972c9301/C in TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:05,596 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/ed4bc1ec726c45b291d6157aef868697, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/76681ffa14f64e1ab426423f953997f2, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/0290395aa7714988979eaed5603832dd] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp, totalSize=36.7 K 2024-12-09T23:58:05,596 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting ed4bc1ec726c45b291d6157aef868697, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733788682405 2024-12-09T23:58:05,597 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 76681ffa14f64e1ab426423f953997f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733788682504 2024-12-09T23:58:05,597 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 0290395aa7714988979eaed5603832dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733788683162 2024-12-09T23:58:05,604 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 472b3103632c40151a72730c972c9301#C#compaction#236 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:05,604 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/72d43b237bd74a44891fbf97f0cee94d is 50, key is test_row_0/C:col10/1733788683165/Put/seqid=0 2024-12-09T23:58:05,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742099_1275 (size=13085) 2024-12-09T23:58:05,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-09T23:58:05,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:05,646 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 472b3103632c40151a72730c972c9301 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T23:58:05,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=A 2024-12-09T23:58:05,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:05,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=B 2024-12-09T23:58:05,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:05,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=C 2024-12-09T23:58:05,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:05,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/3678873ca21d46029174ea783f910e7d is 50, key is test_row_0/A:col10/1733788684333/Put/seqid=0 2024-12-09T23:58:05,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742100_1276 (size=12301) 2024-12-09T23:58:05,656 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/3678873ca21d46029174ea783f910e7d 2024-12-09T23:58:05,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/8bf88cf98b504d7db5df62d7c09f05b0 is 50, key is test_row_0/B:col10/1733788684333/Put/seqid=0 2024-12-09T23:58:05,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742101_1277 (size=12301) 2024-12-09T23:58:05,666 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/8bf88cf98b504d7db5df62d7c09f05b0 2024-12-09T23:58:05,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/b99d8e1764c443419bff4a38e59b18fd is 50, key is test_row_0/C:col10/1733788684333/Put/seqid=0 2024-12-09T23:58:05,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742102_1278 (size=12301) 2024-12-09T23:58:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T23:58:06,027 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/90a715e6ccba405ab8722e22e7ae14ff as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/90a715e6ccba405ab8722e22e7ae14ff 2024-12-09T23:58:06,029 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/72d43b237bd74a44891fbf97f0cee94d as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/72d43b237bd74a44891fbf97f0cee94d 2024-12-09T23:58:06,032 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 472b3103632c40151a72730c972c9301/A of 472b3103632c40151a72730c972c9301 into 90a715e6ccba405ab8722e22e7ae14ff(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:06,032 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:58:06,032 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., storeName=472b3103632c40151a72730c972c9301/A, priority=13, startTime=1733788685560; duration=0sec 2024-12-09T23:58:06,032 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:06,032 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 472b3103632c40151a72730c972c9301:A 2024-12-09T23:58:06,034 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 472b3103632c40151a72730c972c9301/C of 472b3103632c40151a72730c972c9301 into 72d43b237bd74a44891fbf97f0cee94d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:06,034 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:58:06,035 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., storeName=472b3103632c40151a72730c972c9301/C, priority=13, startTime=1733788685560; duration=0sec 2024-12-09T23:58:06,035 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:06,035 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 472b3103632c40151a72730c972c9301:C 2024-12-09T23:58:06,093 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/b99d8e1764c443419bff4a38e59b18fd 2024-12-09T23:58:06,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/3678873ca21d46029174ea783f910e7d as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/3678873ca21d46029174ea783f910e7d 2024-12-09T23:58:06,102 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/3678873ca21d46029174ea783f910e7d, entries=150, sequenceid=353, filesize=12.0 K 2024-12-09T23:58:06,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/8bf88cf98b504d7db5df62d7c09f05b0 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/8bf88cf98b504d7db5df62d7c09f05b0 2024-12-09T23:58:06,107 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/8bf88cf98b504d7db5df62d7c09f05b0, entries=150, sequenceid=353, filesize=12.0 K 2024-12-09T23:58:06,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/b99d8e1764c443419bff4a38e59b18fd as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/b99d8e1764c443419bff4a38e59b18fd 2024-12-09T23:58:06,112 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/b99d8e1764c443419bff4a38e59b18fd, entries=150, sequenceid=353, filesize=12.0 K 2024-12-09T23:58:06,113 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 472b3103632c40151a72730c972c9301 in 467ms, sequenceid=353, compaction requested=false 2024-12-09T23:58:06,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:58:06,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:06,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-09T23:58:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-09T23:58:06,115 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-09T23:58:06,116 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5390 sec 2024-12-09T23:58:06,117 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.5430 sec 2024-12-09T23:58:06,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on 472b3103632c40151a72730c972c9301 2024-12-09T23:58:06,494 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 472b3103632c40151a72730c972c9301 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T23:58:06,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=A 2024-12-09T23:58:06,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:06,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=B 2024-12-09T23:58:06,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:06,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=C 2024-12-09T23:58:06,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:06,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/35dbdb23dc1a436ebc6fa2427f904016 is 50, key is test_row_0/A:col10/1733788686493/Put/seqid=0 2024-12-09T23:58:06,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742103_1279 (size=14741) 2024-12-09T23:58:06,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/35dbdb23dc1a436ebc6fa2427f904016 2024-12-09T23:58:06,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/51dd33a0439547c68cba49b25c71a4db is 50, key is test_row_0/B:col10/1733788686493/Put/seqid=0 2024-12-09T23:58:06,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742104_1280 (size=12301) 2024-12-09T23:58:06,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:06,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788746529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:06,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733788686512, max timestamp=1733788686531Num uniques 859; 2024-12-09T23:58:06,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733788686512, max timestamp=1733788686531Num uniques 859; 2024-12-09T23:58:06,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733788686512, max timestamp=1733788686531Num uniques 859; 2024-12-09T23:58:06,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T23:58:06,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T23:58:06,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T23:58:06,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:06,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:06,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:06,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:06,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788746586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:06,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:06,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:06,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:06,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788746642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:06,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:06,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:06,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:06,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:06,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788746693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:06,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:06,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:06,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:06,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T23:58:06,723 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:58:06,723 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:58:06,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-09T23:58:06,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T23:58:06,724 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:58:06,725 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:58:06,725 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:58:06,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T23:58:06,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:06,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788746852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:06,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:06,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:06,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:06,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-09T23:58:06,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:06,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:58:06,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:06,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:06,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:06,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:06,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788746903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:06,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:06,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:06,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:06,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/51dd33a0439547c68cba49b25c71a4db 2024-12-09T23:58:06,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/c19decfbd82b45559385d832f27fa653 is 50, key is test_row_0/C:col10/1733788686493/Put/seqid=0 2024-12-09T23:58:06,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742105_1281 (size=12301) 2024-12-09T23:58:07,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-09T23:58:07,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:07,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:58:07,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:07,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:07,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:07,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:07,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T23:58:07,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:07,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788747163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:07,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:07,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:07,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:07,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-09T23:58:07,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:07,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:58:07,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:07,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:07,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:07,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788747212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:07,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:07,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:07,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:07,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/c19decfbd82b45559385d832f27fa653 2024-12-09T23:58:07,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/35dbdb23dc1a436ebc6fa2427f904016 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/35dbdb23dc1a436ebc6fa2427f904016 2024-12-09T23:58:07,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/35dbdb23dc1a436ebc6fa2427f904016, entries=200, sequenceid=366, filesize=14.4 K 2024-12-09T23:58:07,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-09T23:58:07,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/51dd33a0439547c68cba49b25c71a4db as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/51dd33a0439547c68cba49b25c71a4db 2024-12-09T23:58:07,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:07,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:58:07,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:07,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:07,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:07,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:07,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/51dd33a0439547c68cba49b25c71a4db, entries=150, sequenceid=366, filesize=12.0 K 2024-12-09T23:58:07,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/c19decfbd82b45559385d832f27fa653 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/c19decfbd82b45559385d832f27fa653 2024-12-09T23:58:07,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/c19decfbd82b45559385d832f27fa653, entries=150, sequenceid=366, filesize=12.0 K 2024-12-09T23:58:07,347 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 472b3103632c40151a72730c972c9301 in 853ms, sequenceid=366, compaction requested=true 2024-12-09T23:58:07,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:58:07,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 472b3103632c40151a72730c972c9301:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:58:07,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:07,347 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:58:07,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 472b3103632c40151a72730c972c9301:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:58:07,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:07,347 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:58:07,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 472b3103632c40151a72730c972c9301:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:58:07,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:58:07,348 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:58:07,348 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): 472b3103632c40151a72730c972c9301/A is initiating minor compaction (all files) 2024-12-09T23:58:07,348 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 472b3103632c40151a72730c972c9301/A in TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:07,348 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/90a715e6ccba405ab8722e22e7ae14ff, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/3678873ca21d46029174ea783f910e7d, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/35dbdb23dc1a436ebc6fa2427f904016] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp, totalSize=39.2 K 2024-12-09T23:58:07,348 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:58:07,349 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): 472b3103632c40151a72730c972c9301/B is initiating minor compaction (all files) 2024-12-09T23:58:07,349 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 472b3103632c40151a72730c972c9301/B in TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:07,349 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/7fcfab5c201a4125bf9366088c77a00d, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/8bf88cf98b504d7db5df62d7c09f05b0, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/51dd33a0439547c68cba49b25c71a4db] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp, totalSize=36.8 K 2024-12-09T23:58:07,349 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90a715e6ccba405ab8722e22e7ae14ff, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733788683162 2024-12-09T23:58:07,349 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3678873ca21d46029174ea783f910e7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733788684326 2024-12-09T23:58:07,350 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fcfab5c201a4125bf9366088c77a00d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733788683162 2024-12-09T23:58:07,350 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35dbdb23dc1a436ebc6fa2427f904016, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1733788686488 2024-12-09T23:58:07,350 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bf88cf98b504d7db5df62d7c09f05b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733788684326 2024-12-09T23:58:07,350 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 51dd33a0439547c68cba49b25c71a4db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1733788686491 2024-12-09T23:58:07,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T23:58:07,365 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 472b3103632c40151a72730c972c9301#A#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:07,366 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/cb47a41c12b44352b61f1a6b62fb48b3 is 50, key is test_row_0/A:col10/1733788686493/Put/seqid=0 2024-12-09T23:58:07,373 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 472b3103632c40151a72730c972c9301#B#compaction#244 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:07,374 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/9fb408ebe0be41a2ad747c8479fcf62a is 50, key is test_row_0/B:col10/1733788686493/Put/seqid=0 2024-12-09T23:58:07,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742107_1283 (size=13187) 2024-12-09T23:58:07,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742106_1282 (size=13187) 2024-12-09T23:58:07,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-09T23:58:07,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:07,488 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 472b3103632c40151a72730c972c9301 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-09T23:58:07,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=A 2024-12-09T23:58:07,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:58:07,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=B 2024-12-09T23:58:07,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:58:07,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=C 2024-12-09T23:58:07,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:58:07,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/dad90c55de814d248ffc5d03a4acdbd8 is 50, key is test_row_0/A:col10/1733788686581/Put/seqid=0 2024-12-09T23:58:07,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742108_1284 (size=12301) 2024-12-09T23:58:07,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on 472b3103632c40151a72730c972c9301 2024-12-09T23:58:07,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. as already flushing 2024-12-09T23:58:07,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:07,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788747709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:07,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:07,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:07,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:07,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:07,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788747722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:07,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:07,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:07,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:07,788 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/9fb408ebe0be41a2ad747c8479fcf62a as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/9fb408ebe0be41a2ad747c8479fcf62a 2024-12-09T23:58:07,793 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 472b3103632c40151a72730c972c9301/B of 472b3103632c40151a72730c972c9301 into 9fb408ebe0be41a2ad747c8479fcf62a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:07,793 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:58:07,793 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., storeName=472b3103632c40151a72730c972c9301/B, priority=13, startTime=1733788687347; duration=0sec 2024-12-09T23:58:07,793 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:58:07,793 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 472b3103632c40151a72730c972c9301:B 2024-12-09T23:58:07,793 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:58:07,794 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:58:07,794 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): 472b3103632c40151a72730c972c9301/C is initiating minor compaction (all files) 2024-12-09T23:58:07,794 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 472b3103632c40151a72730c972c9301/C in TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:07,794 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/72d43b237bd74a44891fbf97f0cee94d, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/b99d8e1764c443419bff4a38e59b18fd, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/c19decfbd82b45559385d832f27fa653] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp, totalSize=36.8 K 2024-12-09T23:58:07,794 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 72d43b237bd74a44891fbf97f0cee94d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733788683162 2024-12-09T23:58:07,794 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting b99d8e1764c443419bff4a38e59b18fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733788684326 2024-12-09T23:58:07,795 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting c19decfbd82b45559385d832f27fa653, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1733788686491 2024-12-09T23:58:07,796 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/cb47a41c12b44352b61f1a6b62fb48b3 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/cb47a41c12b44352b61f1a6b62fb48b3 2024-12-09T23:58:07,800 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 472b3103632c40151a72730c972c9301#C#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:07,801 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 472b3103632c40151a72730c972c9301/A of 472b3103632c40151a72730c972c9301 into cb47a41c12b44352b61f1a6b62fb48b3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:07,801 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:58:07,801 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., storeName=472b3103632c40151a72730c972c9301/A, priority=13, startTime=1733788687347; duration=0sec 2024-12-09T23:58:07,801 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:07,801 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 472b3103632c40151a72730c972c9301:A 2024-12-09T23:58:07,801 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/3b101c5668ba4d678b881431f1a53d34 is 50, key is test_row_0/C:col10/1733788686493/Put/seqid=0 2024-12-09T23:58:07,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742109_1285 (size=13187) 2024-12-09T23:58:07,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:07,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788747822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:07,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:07,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:07,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:07,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T23:58:07,897 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/dad90c55de814d248ffc5d03a4acdbd8 2024-12-09T23:58:07,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/65db06db583b452792c79c8178ab17ba is 50, key is test_row_0/B:col10/1733788686581/Put/seqid=0 2024-12-09T23:58:07,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742110_1286 (size=12301) 2024-12-09T23:58:08,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:08,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788748033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:08,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:08,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:08,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:08,216 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/3b101c5668ba4d678b881431f1a53d34 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/3b101c5668ba4d678b881431f1a53d34 2024-12-09T23:58:08,221 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 472b3103632c40151a72730c972c9301/C of 472b3103632c40151a72730c972c9301 into 3b101c5668ba4d678b881431f1a53d34(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:08,221 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:58:08,221 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., storeName=472b3103632c40151a72730c972c9301/C, priority=13, startTime=1733788687347; duration=0sec 2024-12-09T23:58:08,221 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:08,221 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 472b3103632c40151a72730c972c9301:C 2024-12-09T23:58:08,292 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T23:58:08,308 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/65db06db583b452792c79c8178ab17ba 2024-12-09T23:58:08,312 INFO [Thread-985 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-985. 2024-12-09T23:58:08,312 DEBUG [Thread-985 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:58:08,312 DEBUG [Thread-985 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:08,312 DEBUG [Thread-985 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:08,312 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:58:08,312 INFO [Thread-987 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-987. 2024-12-09T23:58:08,312 DEBUG [Thread-987 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:58:08,312 DEBUG [Thread-987 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:08,312 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:58:08,313 DEBUG [Thread-987 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:08,315 INFO [Thread-989 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-989. 2024-12-09T23:58:08,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/07b973ccf3d64ecb8c36937de5c0b26f is 50, key is test_row_0/C:col10/1733788686581/Put/seqid=0 2024-12-09T23:58:08,315 DEBUG [Thread-989 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:58:08,315 DEBUG [Thread-989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:08,315 INFO [Thread-983 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-983. 2024-12-09T23:58:08,315 DEBUG [Thread-989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:08,315 DEBUG [Thread-983 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:58:08,315 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:58:08,315 DEBUG [Thread-983 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:08,315 DEBUG [Thread-983 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:08,315 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:58:08,316 INFO [Thread-981 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-981. 2024-12-09T23:58:08,316 DEBUG [Thread-981 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:58:08,316 DEBUG [Thread-981 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:08,316 DEBUG [Thread-981 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:08,316 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:58:08,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742111_1287 (size=12301) 2024-12-09T23:58:08,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:08,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1733788748343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:08,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:08,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:08,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:08,720 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/07b973ccf3d64ecb8c36937de5c0b26f 2024-12-09T23:58:08,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/dad90c55de814d248ffc5d03a4acdbd8 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/dad90c55de814d248ffc5d03a4acdbd8 2024-12-09T23:58:08,737 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/dad90c55de814d248ffc5d03a4acdbd8, entries=150, sequenceid=402, filesize=12.0 K 2024-12-09T23:58:08,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/65db06db583b452792c79c8178ab17ba as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/65db06db583b452792c79c8178ab17ba 2024-12-09T23:58:08,742 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/65db06db583b452792c79c8178ab17ba, entries=150, sequenceid=402, filesize=12.0 K 2024-12-09T23:58:08,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/07b973ccf3d64ecb8c36937de5c0b26f as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/07b973ccf3d64ecb8c36937de5c0b26f 2024-12-09T23:58:08,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:08,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48718 deadline: 1733788748743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:08,744 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_0' in 472b3103632c40151a72730c972c9301 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:58:08,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:08,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:08,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:08,747 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/07b973ccf3d64ecb8c36937de5c0b26f, entries=150, sequenceid=402, filesize=12.0 K 2024-12-09T23:58:08,748 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 472b3103632c40151a72730c972c9301 in 1260ms, sequenceid=402, compaction requested=false 2024-12-09T23:58:08,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:58:08,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:08,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-09T23:58:08,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-09T23:58:08,751 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-09T23:58:08,751 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0240 sec 2024-12-09T23:58:08,752 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.0280 sec 2024-12-09T23:58:08,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on 472b3103632c40151a72730c972c9301 2024-12-09T23:58:08,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 472b3103632c40151a72730c972c9301 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T23:58:08,784 INFO [Thread-978 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-978. 2024-12-09T23:58:08,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=A 2024-12-09T23:58:08,784 DEBUG [Thread-978 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:58:08,784 DEBUG [Thread-978 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:08,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:08,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=B 2024-12-09T23:58:08,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:08,784 DEBUG [Thread-978 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:08,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=C 2024-12-09T23:58:08,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:08,784 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:58:08,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/490ab276bb9749f4b47b557667b0d8ba is 50, key is test_row_0/A:col10/1733788687694/Put/seqid=0 2024-12-09T23:58:08,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742112_1288 (size=12301) 2024-12-09T23:58:08,857 INFO [Thread-972 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-972. 2024-12-09T23:58:08,857 DEBUG [Thread-972 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:58:08,857 DEBUG [Thread-972 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:08,857 DEBUG [Thread-972 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:08,857 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:58:08,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T23:58:08,874 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:58:08,876 INFO [Thread-974 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-974. 2024-12-09T23:58:08,876 INFO [Thread-970 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-970. 2024-12-09T23:58:08,876 DEBUG [Thread-974 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:58:08,876 DEBUG [Thread-974 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:08,876 DEBUG [Thread-970 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:58:08,876 DEBUG [Thread-970 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:08,876 DEBUG [Thread-974 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:08,876 DEBUG [Thread-970 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:08,876 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:58:08,876 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:58:09,193 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/490ab276bb9749f4b47b557667b0d8ba 2024-12-09T23:58:09,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/b6208c290a2c46b68ad15f36a8855f76 is 50, key is test_row_0/B:col10/1733788687694/Put/seqid=0 2024-12-09T23:58:09,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742113_1289 (size=12301) 2024-12-09T23:58:09,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/b6208c290a2c46b68ad15f36a8855f76 2024-12-09T23:58:09,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/7e7f3e08a0e94f279accdeb93d5fbb7d is 50, key is test_row_0/C:col10/1733788687694/Put/seqid=0 2024-12-09T23:58:09,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742114_1290 (size=12301) 2024-12-09T23:58:10,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/7e7f3e08a0e94f279accdeb93d5fbb7d 2024-12-09T23:58:10,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/490ab276bb9749f4b47b557667b0d8ba as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/490ab276bb9749f4b47b557667b0d8ba 2024-12-09T23:58:10,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/490ab276bb9749f4b47b557667b0d8ba, entries=150, sequenceid=417, filesize=12.0 K 2024-12-09T23:58:10,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/b6208c290a2c46b68ad15f36a8855f76 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/b6208c290a2c46b68ad15f36a8855f76 2024-12-09T23:58:10,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/b6208c290a2c46b68ad15f36a8855f76, entries=150, sequenceid=417, filesize=12.0 K 2024-12-09T23:58:10,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/7e7f3e08a0e94f279accdeb93d5fbb7d as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/7e7f3e08a0e94f279accdeb93d5fbb7d 2024-12-09T23:58:10,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/7e7f3e08a0e94f279accdeb93d5fbb7d, entries=150, sequenceid=417, filesize=12.0 K 2024-12-09T23:58:10,060 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=20.13 KB/20610 for 472b3103632c40151a72730c972c9301 in 1276ms, sequenceid=417, compaction requested=true 2024-12-09T23:58:10,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:58:10,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 472b3103632c40151a72730c972c9301:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:58:10,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:10,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 472b3103632c40151a72730c972c9301:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:58:10,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:10,060 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:58:10,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 472b3103632c40151a72730c972c9301:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:58:10,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:58:10,060 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:58:10,061 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:58:10,061 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:58:10,061 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): 472b3103632c40151a72730c972c9301/A is initiating minor compaction (all files) 2024-12-09T23:58:10,061 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): 472b3103632c40151a72730c972c9301/B is initiating minor compaction (all files) 2024-12-09T23:58:10,061 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 472b3103632c40151a72730c972c9301/B in TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:10,061 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 472b3103632c40151a72730c972c9301/A in TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:10,061 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/9fb408ebe0be41a2ad747c8479fcf62a, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/65db06db583b452792c79c8178ab17ba, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/b6208c290a2c46b68ad15f36a8855f76] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp, totalSize=36.9 K 2024-12-09T23:58:10,061 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/cb47a41c12b44352b61f1a6b62fb48b3, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/dad90c55de814d248ffc5d03a4acdbd8, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/490ab276bb9749f4b47b557667b0d8ba] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp, totalSize=36.9 K 2024-12-09T23:58:10,061 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fb408ebe0be41a2ad747c8479fcf62a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1733788686491 2024-12-09T23:58:10,061 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb47a41c12b44352b61f1a6b62fb48b3, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1733788686491 2024-12-09T23:58:10,061 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 65db06db583b452792c79c8178ab17ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733788686578 2024-12-09T23:58:10,061 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting dad90c55de814d248ffc5d03a4acdbd8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733788686578 2024-12-09T23:58:10,062 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting b6208c290a2c46b68ad15f36a8855f76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733788687694 2024-12-09T23:58:10,062 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 490ab276bb9749f4b47b557667b0d8ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733788687694 2024-12-09T23:58:10,069 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 472b3103632c40151a72730c972c9301#B#compaction#253 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:10,069 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 472b3103632c40151a72730c972c9301#A#compaction#252 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:10,069 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/6008af4dab934164a355485604712081 is 50, key is test_row_0/A:col10/1733788687694/Put/seqid=0 2024-12-09T23:58:10,069 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/953d6bc396774a7597becac90b87a5d2 is 50, key is test_row_0/B:col10/1733788687694/Put/seqid=0 2024-12-09T23:58:10,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742115_1291 (size=13289) 2024-12-09T23:58:10,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742116_1292 (size=13289) 2024-12-09T23:58:10,486 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/6008af4dab934164a355485604712081 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/6008af4dab934164a355485604712081 2024-12-09T23:58:10,486 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/953d6bc396774a7597becac90b87a5d2 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/953d6bc396774a7597becac90b87a5d2 2024-12-09T23:58:10,491 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 472b3103632c40151a72730c972c9301/B of 472b3103632c40151a72730c972c9301 into 953d6bc396774a7597becac90b87a5d2(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:10,491 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 472b3103632c40151a72730c972c9301/A of 472b3103632c40151a72730c972c9301 into 6008af4dab934164a355485604712081(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:10,491 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:58:10,491 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:58:10,491 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., storeName=472b3103632c40151a72730c972c9301/B, priority=13, startTime=1733788690060; duration=0sec 2024-12-09T23:58:10,491 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., storeName=472b3103632c40151a72730c972c9301/A, priority=13, startTime=1733788690060; duration=0sec 2024-12-09T23:58:10,491 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:58:10,491 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 472b3103632c40151a72730c972c9301:B 2024-12-09T23:58:10,491 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:10,491 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:58:10,491 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 472b3103632c40151a72730c972c9301:A 2024-12-09T23:58:10,492 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:58:10,492 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): 472b3103632c40151a72730c972c9301/C is initiating minor compaction (all files) 2024-12-09T23:58:10,492 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 472b3103632c40151a72730c972c9301/C in TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:10,492 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/3b101c5668ba4d678b881431f1a53d34, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/07b973ccf3d64ecb8c36937de5c0b26f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/7e7f3e08a0e94f279accdeb93d5fbb7d] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp, totalSize=36.9 K 2024-12-09T23:58:10,493 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b101c5668ba4d678b881431f1a53d34, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1733788686491 2024-12-09T23:58:10,493 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 07b973ccf3d64ecb8c36937de5c0b26f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733788686578 2024-12-09T23:58:10,493 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e7f3e08a0e94f279accdeb93d5fbb7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733788687694 2024-12-09T23:58:10,500 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 472b3103632c40151a72730c972c9301#C#compaction#254 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:10,501 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/6f62c723abd54ff39573fbcfc699cab6 is 50, key is test_row_0/C:col10/1733788687694/Put/seqid=0 2024-12-09T23:58:10,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742117_1293 (size=13289) 2024-12-09T23:58:10,757 INFO [Thread-976 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-976. 2024-12-09T23:58:10,757 DEBUG [Thread-976 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:58:10,757 DEBUG [Thread-976 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:10,757 DEBUG [Thread-976 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:10,757 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:58:10,758 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T23:58:10,758 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-12-09T23:58:10,758 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 131 2024-12-09T23:58:10,758 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-12-09T23:58:10,758 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 153 2024-12-09T23:58:10,758 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-12-09T23:58:10,758 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T23:58:10,758 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4798 2024-12-09T23:58:10,758 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4804 2024-12-09T23:58:10,758 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4722 2024-12-09T23:58:10,758 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4801 2024-12-09T23:58:10,758 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4779 2024-12-09T23:58:10,758 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T23:58:10,758 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T23:58:10,759 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T23:58:10,759 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:10,759 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:10,759 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:58:10,761 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-09T23:58:10,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T23:58:10,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T23:58:10,766 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733788690766"}]},"ts":"1733788690766"} 2024-12-09T23:58:10,768 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T23:58:10,768 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T23:58:10,769 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T23:58:10,770 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=472b3103632c40151a72730c972c9301, UNASSIGN}] 2024-12-09T23:58:10,771 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=472b3103632c40151a72730c972c9301, UNASSIGN 2024-12-09T23:58:10,772 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=472b3103632c40151a72730c972c9301, regionState=CLOSING, regionLocation=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:10,773 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=472b3103632c40151a72730c972c9301, UNASSIGN because future has completed 2024-12-09T23:58:10,774 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T23:58:10,774 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure 472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837}] 2024-12-09T23:58:10,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T23:58:10,915 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/6f62c723abd54ff39573fbcfc699cab6 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/6f62c723abd54ff39573fbcfc699cab6 2024-12-09T23:58:10,922 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 472b3103632c40151a72730c972c9301/C of 472b3103632c40151a72730c972c9301 into 6f62c723abd54ff39573fbcfc699cab6(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:10,922 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 472b3103632c40151a72730c972c9301: 2024-12-09T23:58:10,922 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., storeName=472b3103632c40151a72730c972c9301/C, priority=13, startTime=1733788690060; duration=0sec 2024-12-09T23:58:10,922 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:10,922 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 472b3103632c40151a72730c972c9301:C 2024-12-09T23:58:10,926 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(122): Close 472b3103632c40151a72730c972c9301 2024-12-09T23:58:10,926 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T23:58:10,926 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1722): Closing 472b3103632c40151a72730c972c9301, disabling compactions & flushes 2024-12-09T23:58:10,926 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:10,926 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:10,926 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. after waiting 0 ms 2024-12-09T23:58:10,926 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:10,926 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(2902): Flushing 472b3103632c40151a72730c972c9301 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-09T23:58:10,926 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=A 2024-12-09T23:58:10,926 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:10,926 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=B 2024-12-09T23:58:10,926 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:10,926 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 472b3103632c40151a72730c972c9301, store=C 2024-12-09T23:58:10,926 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:10,929 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/5a4c32b8fac6436781a7f371be2e5f0f is 50, key is test_row_0/A:col10/1733788690754/Put/seqid=0 2024-12-09T23:58:10,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742118_1294 (size=12301) 2024-12-09T23:58:11,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T23:58:11,334 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/5a4c32b8fac6436781a7f371be2e5f0f 2024-12-09T23:58:11,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T23:58:11,438 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/5a949d1a15d1439280e5908db68c3289 is 50, key is test_row_0/B:col10/1733788690754/Put/seqid=0 2024-12-09T23:58:11,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742119_1295 (size=12301) 2024-12-09T23:58:11,844 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/5a949d1a15d1439280e5908db68c3289 2024-12-09T23:58:11,856 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/b0eb63b895024d109863c250e36c7871 is 50, key is test_row_0/C:col10/1733788690754/Put/seqid=0 2024-12-09T23:58:11,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742120_1296 (size=12301) 2024-12-09T23:58:11,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T23:58:12,263 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/b0eb63b895024d109863c250e36c7871 2024-12-09T23:58:12,276 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/A/5a4c32b8fac6436781a7f371be2e5f0f as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/5a4c32b8fac6436781a7f371be2e5f0f 2024-12-09T23:58:12,280 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/5a4c32b8fac6436781a7f371be2e5f0f, entries=150, sequenceid=427, filesize=12.0 K 2024-12-09T23:58:12,281 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/B/5a949d1a15d1439280e5908db68c3289 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/5a949d1a15d1439280e5908db68c3289 2024-12-09T23:58:12,284 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/5a949d1a15d1439280e5908db68c3289, entries=150, sequenceid=427, filesize=12.0 K 2024-12-09T23:58:12,285 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/.tmp/C/b0eb63b895024d109863c250e36c7871 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/b0eb63b895024d109863c250e36c7871 2024-12-09T23:58:12,288 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/b0eb63b895024d109863c250e36c7871, entries=150, sequenceid=427, filesize=12.0 K 2024-12-09T23:58:12,289 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 472b3103632c40151a72730c972c9301 in 1363ms, sequenceid=427, compaction requested=false 2024-12-09T23:58:12,289 DEBUG [StoreCloser-TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/cfecc2c7d67b432baa19ad68c6318985, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/0bb3028debf14d57ae38fc60332c1d77, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/d0982414e95c4ea08f7877c23d5863b3, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/aa6490637aa648198867f224cffea16b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/6a8f2c92e9ec49be8b9d5000200d89bd, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/98ec75e163ac422e927b1bd18adac973, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/80839f8f625440a980269c155f7f8c14, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/8cc9c130d6ff40b4865e36bf9bf87745, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/e4a72d69c2ae425a8074737527eebd84, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/80bd5dff162d4dd8bbb38fe24e82ea71, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/8f2b765900da41099f7d292be536035f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/bf075875ccae46c0af73bfa7ffc57b9d, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/7b2d8e3699be4404a70c1d604d74c735, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/be01d367738c4bb89f8cc242137d920c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/b457e0ba898a4fb08625c78b1b4cbcf8, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/71deabdc62af4cdc9fe55c4299cbb51f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/476d2b1b10d44a278ac944bc7c9066b4, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/c588ebdf210143e782f98aee727a084e, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/be2ccff95b7349d2a04b5efca4746d63, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/a17fb33618a645f799c296a124260e8f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/7c5b1c55d7c5446a94d414adfa9d8e1f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/c0dfca9e6f7443c0a76402125fb2aedd, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/95e2142f7a054299a0ec8ce3090debe2, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/90a715e6ccba405ab8722e22e7ae14ff, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/3678873ca21d46029174ea783f910e7d, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/35dbdb23dc1a436ebc6fa2427f904016, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/cb47a41c12b44352b61f1a6b62fb48b3, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/dad90c55de814d248ffc5d03a4acdbd8, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/490ab276bb9749f4b47b557667b0d8ba] to archive 2024-12-09T23:58:12,290 DEBUG [StoreCloser-TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-09T23:58:12,292 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/0bb3028debf14d57ae38fc60332c1d77 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/0bb3028debf14d57ae38fc60332c1d77 2024-12-09T23:58:12,292 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/d0982414e95c4ea08f7877c23d5863b3 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/d0982414e95c4ea08f7877c23d5863b3 2024-12-09T23:58:12,292 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/aa6490637aa648198867f224cffea16b to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/aa6490637aa648198867f224cffea16b 2024-12-09T23:58:12,292 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/cfecc2c7d67b432baa19ad68c6318985 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/cfecc2c7d67b432baa19ad68c6318985 2024-12-09T23:58:12,292 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/6a8f2c92e9ec49be8b9d5000200d89bd to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/6a8f2c92e9ec49be8b9d5000200d89bd 2024-12-09T23:58:12,293 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/80839f8f625440a980269c155f7f8c14 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/80839f8f625440a980269c155f7f8c14 2024-12-09T23:58:12,293 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/98ec75e163ac422e927b1bd18adac973 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/98ec75e163ac422e927b1bd18adac973 2024-12-09T23:58:12,293 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/8cc9c130d6ff40b4865e36bf9bf87745 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/8cc9c130d6ff40b4865e36bf9bf87745 2024-12-09T23:58:12,294 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/e4a72d69c2ae425a8074737527eebd84 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/e4a72d69c2ae425a8074737527eebd84 2024-12-09T23:58:12,294 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/80bd5dff162d4dd8bbb38fe24e82ea71 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/80bd5dff162d4dd8bbb38fe24e82ea71 2024-12-09T23:58:12,294 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/8f2b765900da41099f7d292be536035f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/8f2b765900da41099f7d292be536035f 2024-12-09T23:58:12,294 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/be01d367738c4bb89f8cc242137d920c to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/be01d367738c4bb89f8cc242137d920c 2024-12-09T23:58:12,295 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/bf075875ccae46c0af73bfa7ffc57b9d to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/bf075875ccae46c0af73bfa7ffc57b9d 2024-12-09T23:58:12,295 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/7b2d8e3699be4404a70c1d604d74c735 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/7b2d8e3699be4404a70c1d604d74c735 2024-12-09T23:58:12,295 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/71deabdc62af4cdc9fe55c4299cbb51f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/71deabdc62af4cdc9fe55c4299cbb51f 2024-12-09T23:58:12,296 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/b457e0ba898a4fb08625c78b1b4cbcf8 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/b457e0ba898a4fb08625c78b1b4cbcf8 2024-12-09T23:58:12,296 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/c588ebdf210143e782f98aee727a084e to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/c588ebdf210143e782f98aee727a084e 2024-12-09T23:58:12,297 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/a17fb33618a645f799c296a124260e8f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/a17fb33618a645f799c296a124260e8f 2024-12-09T23:58:12,297 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/476d2b1b10d44a278ac944bc7c9066b4 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/476d2b1b10d44a278ac944bc7c9066b4 2024-12-09T23:58:12,297 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/7c5b1c55d7c5446a94d414adfa9d8e1f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/7c5b1c55d7c5446a94d414adfa9d8e1f 2024-12-09T23:58:12,297 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/be2ccff95b7349d2a04b5efca4746d63 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/be2ccff95b7349d2a04b5efca4746d63 2024-12-09T23:58:12,297 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/95e2142f7a054299a0ec8ce3090debe2 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/95e2142f7a054299a0ec8ce3090debe2 2024-12-09T23:58:12,297 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/c0dfca9e6f7443c0a76402125fb2aedd to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/c0dfca9e6f7443c0a76402125fb2aedd 2024-12-09T23:58:12,298 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/90a715e6ccba405ab8722e22e7ae14ff to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/90a715e6ccba405ab8722e22e7ae14ff 2024-12-09T23:58:12,298 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/35dbdb23dc1a436ebc6fa2427f904016 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/35dbdb23dc1a436ebc6fa2427f904016 2024-12-09T23:58:12,298 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/3678873ca21d46029174ea783f910e7d to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/3678873ca21d46029174ea783f910e7d 2024-12-09T23:58:12,299 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/cb47a41c12b44352b61f1a6b62fb48b3 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/cb47a41c12b44352b61f1a6b62fb48b3 2024-12-09T23:58:12,299 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/490ab276bb9749f4b47b557667b0d8ba to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/490ab276bb9749f4b47b557667b0d8ba 2024-12-09T23:58:12,299 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/dad90c55de814d248ffc5d03a4acdbd8 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/dad90c55de814d248ffc5d03a4acdbd8 2024-12-09T23:58:12,301 DEBUG [StoreCloser-TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/928334453a084a708ce7eae376b852b5, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/1c17a6c6813d49dfb12f53cd2001f3ff, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/92180db52aaf45c2aae651a0fe32c66f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/e6875f0d5221416ea9e398a61457af3c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/836f248d82804c22be0303b473f9620a, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/57c01c3d210b4cb9bb572bff8f61cb57, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/db83e47692c0419e89287f04e3883fd4, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/f1cc363040914da681b1dd241901aa99, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/7e5ebc4ce174466ab71b55490d5aa764, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/2951f28d72a64b3289e77363674dd1e8, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/3a55c877d75d470abe73a7b3e747147f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/5a4ba68dd5fa4adda529da3793b65639, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/b7b55eecf5144f9da89ddbae14405708, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/5d3e4c2609e24214b4961524cd4492c8, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/46f2cdf369da493bb23821740c722749, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/7592595a0b1b48febd32b3272540fb80, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/69265fec8d1f40779b0b5a7b6484a977, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/f1353beb85da4e7095b1fe8af1780d12, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/4f3958c6352c4866b41c1bb7fdc236e3, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/3a6b70d86d3d488a83a032cfff5b402a, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/cad4ba9e52404f9b90317bd150444851, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/ca4f7107693440fc9e822290af0a797d, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/7fcfab5c201a4125bf9366088c77a00d, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/944ac4c34ea74e9b9993023006b067d4, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/8bf88cf98b504d7db5df62d7c09f05b0, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/9fb408ebe0be41a2ad747c8479fcf62a, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/51dd33a0439547c68cba49b25c71a4db, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/65db06db583b452792c79c8178ab17ba, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/b6208c290a2c46b68ad15f36a8855f76] to archive 2024-12-09T23:58:12,302 DEBUG [StoreCloser-TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-09T23:58:12,303 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/928334453a084a708ce7eae376b852b5 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/928334453a084a708ce7eae376b852b5 2024-12-09T23:58:12,303 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/e6875f0d5221416ea9e398a61457af3c to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/e6875f0d5221416ea9e398a61457af3c 2024-12-09T23:58:12,303 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/836f248d82804c22be0303b473f9620a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/836f248d82804c22be0303b473f9620a 2024-12-09T23:58:12,303 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/92180db52aaf45c2aae651a0fe32c66f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/92180db52aaf45c2aae651a0fe32c66f 2024-12-09T23:58:12,304 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/1c17a6c6813d49dfb12f53cd2001f3ff to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/1c17a6c6813d49dfb12f53cd2001f3ff 2024-12-09T23:58:12,304 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/f1cc363040914da681b1dd241901aa99 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/f1cc363040914da681b1dd241901aa99 2024-12-09T23:58:12,304 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/db83e47692c0419e89287f04e3883fd4 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/db83e47692c0419e89287f04e3883fd4 2024-12-09T23:58:12,304 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/57c01c3d210b4cb9bb572bff8f61cb57 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/57c01c3d210b4cb9bb572bff8f61cb57 2024-12-09T23:58:12,305 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/7e5ebc4ce174466ab71b55490d5aa764 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/7e5ebc4ce174466ab71b55490d5aa764 2024-12-09T23:58:12,305 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/2951f28d72a64b3289e77363674dd1e8 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/2951f28d72a64b3289e77363674dd1e8 2024-12-09T23:58:12,305 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/5a4ba68dd5fa4adda529da3793b65639 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/5a4ba68dd5fa4adda529da3793b65639 2024-12-09T23:58:12,305 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/5d3e4c2609e24214b4961524cd4492c8 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/5d3e4c2609e24214b4961524cd4492c8 2024-12-09T23:58:12,305 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/b7b55eecf5144f9da89ddbae14405708 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/b7b55eecf5144f9da89ddbae14405708 2024-12-09T23:58:12,305 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/46f2cdf369da493bb23821740c722749 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/46f2cdf369da493bb23821740c722749 2024-12-09T23:58:12,306 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/3a55c877d75d470abe73a7b3e747147f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/3a55c877d75d470abe73a7b3e747147f 2024-12-09T23:58:12,306 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/7592595a0b1b48febd32b3272540fb80 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/7592595a0b1b48febd32b3272540fb80 2024-12-09T23:58:12,306 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/69265fec8d1f40779b0b5a7b6484a977 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/69265fec8d1f40779b0b5a7b6484a977 2024-12-09T23:58:12,306 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/f1353beb85da4e7095b1fe8af1780d12 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/f1353beb85da4e7095b1fe8af1780d12 2024-12-09T23:58:12,306 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/3a6b70d86d3d488a83a032cfff5b402a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/3a6b70d86d3d488a83a032cfff5b402a 2024-12-09T23:58:12,306 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/4f3958c6352c4866b41c1bb7fdc236e3 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/4f3958c6352c4866b41c1bb7fdc236e3 2024-12-09T23:58:12,307 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/cad4ba9e52404f9b90317bd150444851 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/cad4ba9e52404f9b90317bd150444851 2024-12-09T23:58:12,307 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/ca4f7107693440fc9e822290af0a797d to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/ca4f7107693440fc9e822290af0a797d 2024-12-09T23:58:12,307 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/7fcfab5c201a4125bf9366088c77a00d to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/7fcfab5c201a4125bf9366088c77a00d 2024-12-09T23:58:12,307 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/944ac4c34ea74e9b9993023006b067d4 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/944ac4c34ea74e9b9993023006b067d4 2024-12-09T23:58:12,308 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/8bf88cf98b504d7db5df62d7c09f05b0 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/8bf88cf98b504d7db5df62d7c09f05b0 2024-12-09T23:58:12,308 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/9fb408ebe0be41a2ad747c8479fcf62a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/9fb408ebe0be41a2ad747c8479fcf62a 2024-12-09T23:58:12,308 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/51dd33a0439547c68cba49b25c71a4db to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/51dd33a0439547c68cba49b25c71a4db 2024-12-09T23:58:12,308 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/65db06db583b452792c79c8178ab17ba to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/65db06db583b452792c79c8178ab17ba 2024-12-09T23:58:12,308 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/b6208c290a2c46b68ad15f36a8855f76 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/b6208c290a2c46b68ad15f36a8855f76 2024-12-09T23:58:12,309 DEBUG [StoreCloser-TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/8175166a8d8d463283eca21fd7241b06, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/fe0e4addc1b54ba4b101e0bf6ab398be, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/cac9b59b13ef41aabe47b5414c8043ac, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/2e96b05db260424f81424df9323d5075, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/a008c9a79b2c4de185c64ba149138ecb, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/2f5e81490f9e4bee92e0714a6f1fb03b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/fb25993b8b04491fa0be05d199044190, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/24c3f76d344a40089a9b6e7a4431ef0f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/02e0a828bf244ba0a110b3091ffdaf5d, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/65793e8165e8459f850d84d696984a49, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/c0b05a09ffd7411f83f839fe51092478, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/ab52f0d20d2144488c49b5d89dc649fc, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/fd743fb5c46f4666a71c9316cbf9103b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/e8079465af9540538ed501d135632cb4, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/ceaf8b2e5e134762b23431bfdcf0e4af, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/ac238f4817d0467dbd1c619e37f5fa9b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/6da666b57dbc4b959acf200d6a81558e, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/9a00a35fe62e4f5b8ac626013b99a2c2, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/b1cf2075d8314d25a790c5034c9aa17c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/ed4bc1ec726c45b291d6157aef868697, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/c6ee59f13a184aefa865010e3ed5ed2c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/76681ffa14f64e1ab426423f953997f2, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/72d43b237bd74a44891fbf97f0cee94d, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/0290395aa7714988979eaed5603832dd, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/b99d8e1764c443419bff4a38e59b18fd, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/3b101c5668ba4d678b881431f1a53d34, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/c19decfbd82b45559385d832f27fa653, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/07b973ccf3d64ecb8c36937de5c0b26f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/7e7f3e08a0e94f279accdeb93d5fbb7d] to archive 2024-12-09T23:58:12,310 DEBUG [StoreCloser-TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-09T23:58:12,312 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/cac9b59b13ef41aabe47b5414c8043ac to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/cac9b59b13ef41aabe47b5414c8043ac 2024-12-09T23:58:12,312 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/2e96b05db260424f81424df9323d5075 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/2e96b05db260424f81424df9323d5075 2024-12-09T23:58:12,312 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/2f5e81490f9e4bee92e0714a6f1fb03b to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/2f5e81490f9e4bee92e0714a6f1fb03b 2024-12-09T23:58:12,312 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/24c3f76d344a40089a9b6e7a4431ef0f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/24c3f76d344a40089a9b6e7a4431ef0f 2024-12-09T23:58:12,312 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/fe0e4addc1b54ba4b101e0bf6ab398be to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/fe0e4addc1b54ba4b101e0bf6ab398be 2024-12-09T23:58:12,312 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/8175166a8d8d463283eca21fd7241b06 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/8175166a8d8d463283eca21fd7241b06 2024-12-09T23:58:12,312 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/a008c9a79b2c4de185c64ba149138ecb to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/a008c9a79b2c4de185c64ba149138ecb 2024-12-09T23:58:12,312 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/fb25993b8b04491fa0be05d199044190 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/fb25993b8b04491fa0be05d199044190 2024-12-09T23:58:12,314 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/02e0a828bf244ba0a110b3091ffdaf5d to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/02e0a828bf244ba0a110b3091ffdaf5d 2024-12-09T23:58:12,314 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/65793e8165e8459f850d84d696984a49 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/65793e8165e8459f850d84d696984a49 2024-12-09T23:58:12,314 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/c0b05a09ffd7411f83f839fe51092478 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/c0b05a09ffd7411f83f839fe51092478 2024-12-09T23:58:12,314 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/ab52f0d20d2144488c49b5d89dc649fc to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/ab52f0d20d2144488c49b5d89dc649fc 2024-12-09T23:58:12,315 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/fd743fb5c46f4666a71c9316cbf9103b to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/fd743fb5c46f4666a71c9316cbf9103b 2024-12-09T23:58:12,315 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/ac238f4817d0467dbd1c619e37f5fa9b to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/ac238f4817d0467dbd1c619e37f5fa9b 2024-12-09T23:58:12,315 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/e8079465af9540538ed501d135632cb4 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/e8079465af9540538ed501d135632cb4 2024-12-09T23:58:12,315 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/ceaf8b2e5e134762b23431bfdcf0e4af to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/ceaf8b2e5e134762b23431bfdcf0e4af 2024-12-09T23:58:12,316 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/b1cf2075d8314d25a790c5034c9aa17c to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/b1cf2075d8314d25a790c5034c9aa17c 2024-12-09T23:58:12,316 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/6da666b57dbc4b959acf200d6a81558e to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/6da666b57dbc4b959acf200d6a81558e 2024-12-09T23:58:12,316 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/ed4bc1ec726c45b291d6157aef868697 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/ed4bc1ec726c45b291d6157aef868697 2024-12-09T23:58:12,316 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/9a00a35fe62e4f5b8ac626013b99a2c2 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/9a00a35fe62e4f5b8ac626013b99a2c2 2024-12-09T23:58:12,316 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/c6ee59f13a184aefa865010e3ed5ed2c to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/c6ee59f13a184aefa865010e3ed5ed2c 2024-12-09T23:58:12,317 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/76681ffa14f64e1ab426423f953997f2 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/76681ffa14f64e1ab426423f953997f2 2024-12-09T23:58:12,317 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/0290395aa7714988979eaed5603832dd to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/0290395aa7714988979eaed5603832dd 2024-12-09T23:58:12,317 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/72d43b237bd74a44891fbf97f0cee94d to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/72d43b237bd74a44891fbf97f0cee94d 2024-12-09T23:58:12,317 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/b99d8e1764c443419bff4a38e59b18fd to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/b99d8e1764c443419bff4a38e59b18fd 2024-12-09T23:58:12,318 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/3b101c5668ba4d678b881431f1a53d34 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/3b101c5668ba4d678b881431f1a53d34 2024-12-09T23:58:12,318 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/c19decfbd82b45559385d832f27fa653 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/c19decfbd82b45559385d832f27fa653 2024-12-09T23:58:12,318 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/7e7f3e08a0e94f279accdeb93d5fbb7d to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/7e7f3e08a0e94f279accdeb93d5fbb7d 2024-12-09T23:58:12,318 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/07b973ccf3d64ecb8c36937de5c0b26f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/07b973ccf3d64ecb8c36937de5c0b26f 2024-12-09T23:58:12,322 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/recovered.edits/430.seqid, newMaxSeqId=430, maxSeqId=1 2024-12-09T23:58:12,322 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. 2024-12-09T23:58:12,322 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1676): Region close journal for 472b3103632c40151a72730c972c9301: Waiting for close lock at 1733788690926Running coprocessor pre-close hooks at 1733788690926Disabling compacts and flushes for region at 1733788690926Disabling writes for close at 1733788690926Obtaining lock to block concurrent updates at 1733788690926Preparing flush snapshotting stores in 472b3103632c40151a72730c972c9301 at 1733788690926Finished memstore snapshotting TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733788690926Flushing stores of TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301. at 1733788690927 (+1 ms)Flushing 472b3103632c40151a72730c972c9301/A: creating writer at 1733788690927Flushing 472b3103632c40151a72730c972c9301/A: appending metadata at 1733788690929 (+2 ms)Flushing 472b3103632c40151a72730c972c9301/A: closing flushed file at 1733788690929Flushing 472b3103632c40151a72730c972c9301/B: creating writer at 1733788691434 (+505 ms)Flushing 472b3103632c40151a72730c972c9301/B: appending metadata at 1733788691438 (+4 ms)Flushing 472b3103632c40151a72730c972c9301/B: closing flushed file at 1733788691438Flushing 472b3103632c40151a72730c972c9301/C: creating writer at 1733788691853 (+415 ms)Flushing 472b3103632c40151a72730c972c9301/C: appending metadata at 1733788691856 (+3 ms)Flushing 472b3103632c40151a72730c972c9301/C: closing flushed file at 1733788691856Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bd2b555: reopening flushed file at 1733788692275 (+419 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@326d5628: reopening flushed file at 1733788692280 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ee1d009: reopening flushed file at 1733788692284 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 472b3103632c40151a72730c972c9301 in 1363ms, sequenceid=427, compaction requested=false at 1733788692289 (+5 ms)Writing region close event to WAL at 1733788692319 (+30 ms)Running coprocessor post-close hooks at 1733788692322 (+3 ms)Closed at 1733788692322 2024-12-09T23:58:12,324 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(157): Closed 472b3103632c40151a72730c972c9301 2024-12-09T23:58:12,324 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=472b3103632c40151a72730c972c9301, regionState=CLOSED 2024-12-09T23:58:12,326 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure 472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 because future has completed 2024-12-09T23:58:12,328 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-09T23:58:12,328 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseRegionProcedure 472b3103632c40151a72730c972c9301, server=2907d75fbb3e,37627,1733788599837 in 1.5520 sec 2024-12-09T23:58:12,329 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-12-09T23:58:12,329 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=472b3103632c40151a72730c972c9301, UNASSIGN in 1.5580 sec 2024-12-09T23:58:12,331 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-09T23:58:12,331 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5610 sec 2024-12-09T23:58:12,332 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733788692332"}]},"ts":"1733788692332"} 2024-12-09T23:58:12,333 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T23:58:12,333 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T23:58:12,334 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5720 sec 2024-12-09T23:58:12,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T23:58:12,914 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:58:12,916 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-09T23:58:12,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T23:58:12,919 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T23:58:12,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-09T23:58:12,919 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=82, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T23:58:12,921 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301 2024-12-09T23:58:12,923 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A, FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B, FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C, FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/recovered.edits] 2024-12-09T23:58:12,926 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/5a4c32b8fac6436781a7f371be2e5f0f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/5a4c32b8fac6436781a7f371be2e5f0f 2024-12-09T23:58:12,927 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/6008af4dab934164a355485604712081 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/A/6008af4dab934164a355485604712081 2024-12-09T23:58:12,930 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/5a949d1a15d1439280e5908db68c3289 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/5a949d1a15d1439280e5908db68c3289 2024-12-09T23:58:12,930 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/953d6bc396774a7597becac90b87a5d2 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/B/953d6bc396774a7597becac90b87a5d2 2024-12-09T23:58:12,934 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/b0eb63b895024d109863c250e36c7871 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/b0eb63b895024d109863c250e36c7871 2024-12-09T23:58:12,934 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/6f62c723abd54ff39573fbcfc699cab6 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/C/6f62c723abd54ff39573fbcfc699cab6 2024-12-09T23:58:12,938 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/recovered.edits/430.seqid to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301/recovered.edits/430.seqid 2024-12-09T23:58:12,939 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/472b3103632c40151a72730c972c9301 2024-12-09T23:58:12,939 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T23:58:12,942 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=82, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T23:58:12,946 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T23:58:12,949 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T23:58:12,951 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=82, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T23:58:12,951 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-09T23:58:12,951 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733788692951"}]},"ts":"9223372036854775807"} 2024-12-09T23:58:12,953 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T23:58:12,953 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 472b3103632c40151a72730c972c9301, NAME => 'TestAcidGuarantees,,1733788666112.472b3103632c40151a72730c972c9301.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T23:58:12,953 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T23:58:12,953 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733788692953"}]},"ts":"9223372036854775807"} 2024-12-09T23:58:12,955 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T23:58:12,956 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=82, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T23:58:12,958 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 40 msec 2024-12-09T23:58:13,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-09T23:58:13,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T23:58:13,024 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:58:13,033 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=258 (was 258), OpenFileDescriptor=461 (was 459) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=362 (was 375), ProcessCount=11 (was 11), AvailableMemoryMB=4007 (was 4028) 2024-12-09T23:58:13,044 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=258, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=362, ProcessCount=11, AvailableMemoryMB=4007 2024-12-09T23:58:13,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T23:58:13,046 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T23:58:13,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T23:58:13,048 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T23:58:13,048 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:58:13,049 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 83 2024-12-09T23:58:13,049 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T23:58:13,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-09T23:58:13,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742121_1297 (size=1061) 2024-12-09T23:58:13,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-09T23:58:13,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-09T23:58:13,458 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e 2024-12-09T23:58:13,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742122_1298 (size=53) 2024-12-09T23:58:13,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-09T23:58:13,869 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:58:13,869 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a41ba930d231212908b11d93742f8c4a, disabling compactions & flushes 2024-12-09T23:58:13,869 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:13,869 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:13,869 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. after waiting 0 ms 2024-12-09T23:58:13,869 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:13,869 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:13,869 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a41ba930d231212908b11d93742f8c4a: Waiting for close lock at 1733788693869Disabling compacts and flushes for region at 1733788693869Disabling writes for close at 1733788693869Writing region close event to WAL at 1733788693869Closed at 1733788693869 2024-12-09T23:58:13,872 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T23:58:13,873 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733788693872"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733788693872"}]},"ts":"1733788693872"} 2024-12-09T23:58:13,877 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T23:58:13,878 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T23:58:13,878 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733788693878"}]},"ts":"1733788693878"} 2024-12-09T23:58:13,881 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T23:58:13,881 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a41ba930d231212908b11d93742f8c4a, ASSIGN}] 2024-12-09T23:58:13,882 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a41ba930d231212908b11d93742f8c4a, ASSIGN 2024-12-09T23:58:13,883 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a41ba930d231212908b11d93742f8c4a, ASSIGN; state=OFFLINE, location=2907d75fbb3e,37627,1733788599837; forceNewPlan=false, retain=false 2024-12-09T23:58:14,034 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=a41ba930d231212908b11d93742f8c4a, regionState=OPENING, regionLocation=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:14,038 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a41ba930d231212908b11d93742f8c4a, ASSIGN because future has completed 2024-12-09T23:58:14,039 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837}] 2024-12-09T23:58:14,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-09T23:58:14,200 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:14,200 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7752): Opening region: {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} 2024-12-09T23:58:14,201 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:14,201 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:58:14,201 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7794): checking encryption for a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:14,201 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7797): checking classloading for a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:14,203 INFO [StoreOpener-a41ba930d231212908b11d93742f8c4a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:14,205 INFO [StoreOpener-a41ba930d231212908b11d93742f8c4a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T23:58:14,205 INFO [StoreOpener-a41ba930d231212908b11d93742f8c4a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a41ba930d231212908b11d93742f8c4a columnFamilyName A 2024-12-09T23:58:14,205 DEBUG [StoreOpener-a41ba930d231212908b11d93742f8c4a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:58:14,206 INFO [StoreOpener-a41ba930d231212908b11d93742f8c4a-1 {}] regionserver.HStore(327): Store=a41ba930d231212908b11d93742f8c4a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:58:14,206 INFO [StoreOpener-a41ba930d231212908b11d93742f8c4a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:14,208 INFO [StoreOpener-a41ba930d231212908b11d93742f8c4a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T23:58:14,208 INFO [StoreOpener-a41ba930d231212908b11d93742f8c4a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a41ba930d231212908b11d93742f8c4a columnFamilyName B 2024-12-09T23:58:14,208 DEBUG [StoreOpener-a41ba930d231212908b11d93742f8c4a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:58:14,209 INFO [StoreOpener-a41ba930d231212908b11d93742f8c4a-1 {}] regionserver.HStore(327): Store=a41ba930d231212908b11d93742f8c4a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:58:14,209 INFO [StoreOpener-a41ba930d231212908b11d93742f8c4a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:14,210 INFO [StoreOpener-a41ba930d231212908b11d93742f8c4a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T23:58:14,211 INFO [StoreOpener-a41ba930d231212908b11d93742f8c4a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a41ba930d231212908b11d93742f8c4a columnFamilyName C 2024-12-09T23:58:14,211 DEBUG [StoreOpener-a41ba930d231212908b11d93742f8c4a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:58:14,211 INFO [StoreOpener-a41ba930d231212908b11d93742f8c4a-1 {}] regionserver.HStore(327): Store=a41ba930d231212908b11d93742f8c4a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:58:14,212 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:14,212 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1038): replaying wal for a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:14,213 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:14,214 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:14,216 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1048): stopping wal replay for a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:14,216 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1060): Cleaning up temporary data for a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:14,216 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T23:58:14,218 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1093): writing seq id for a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:14,221 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T23:58:14,221 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1114): Opened a41ba930d231212908b11d93742f8c4a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66191526, jitterRate=-0.013669401407241821}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T23:58:14,221 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:14,222 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1006): Region open journal for a41ba930d231212908b11d93742f8c4a: Running coprocessor pre-open hook at 1733788694201Writing region info on filesystem at 1733788694201Initializing all the Stores at 1733788694202 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788694202Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788694203 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788694203Cleaning up temporary data from old regions at 1733788694216 (+13 ms)Running coprocessor post-open hooks at 1733788694221 (+5 ms)Region opened successfully at 1733788694222 (+1 ms) 2024-12-09T23:58:14,223 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., pid=85, masterSystemTime=1733788694193 2024-12-09T23:58:14,225 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:14,225 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:14,225 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=a41ba930d231212908b11d93742f8c4a, regionState=OPEN, openSeqNum=2, regionLocation=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:14,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 because future has completed 2024-12-09T23:58:14,230 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-09T23:58:14,230 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 in 189 msec 2024-12-09T23:58:14,232 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-12-09T23:58:14,232 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a41ba930d231212908b11d93742f8c4a, ASSIGN in 349 msec 2024-12-09T23:58:14,233 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T23:58:14,233 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733788694233"}]},"ts":"1733788694233"} 2024-12-09T23:58:14,235 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T23:58:14,236 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T23:58:14,238 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1900 sec 2024-12-09T23:58:15,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-09T23:58:15,195 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:58:15,197 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1774ef6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:15,198 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:58:15,198 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:58:15,200 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:58:15,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:58:15,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:58:15,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26a2e459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:15,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:58:15,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:58:15,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:15,204 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57960, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:58:15,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@108d24d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:15,204 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:58:15,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:58:15,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:58:15,206 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57096, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:58:15,207 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T23:58:15,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 2907d75fbb3e,45191,1733788599119 2024-12-09T23:58:15,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1f57bb87 2024-12-09T23:58:15,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T23:58:15,209 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57968, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T23:58:15,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T23:58:15,211 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T23:58:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-09T23:58:15,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742123_1299 (size=1097) 2024-12-09T23:58:15,626 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-09T23:58:15,626 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-09T23:58:15,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T23:58:15,635 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a41ba930d231212908b11d93742f8c4a, REOPEN/MOVE}] 2024-12-09T23:58:15,636 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a41ba930d231212908b11d93742f8c4a, REOPEN/MOVE 2024-12-09T23:58:15,637 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=a41ba930d231212908b11d93742f8c4a, regionState=CLOSING, regionLocation=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:15,638 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a41ba930d231212908b11d93742f8c4a, REOPEN/MOVE because future has completed 2024-12-09T23:58:15,638 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T23:58:15,638 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837}] 2024-12-09T23:58:15,792 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:15,792 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T23:58:15,792 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing a41ba930d231212908b11d93742f8c4a, disabling compactions & flushes 2024-12-09T23:58:15,792 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:15,792 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:15,792 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. after waiting 0 ms 2024-12-09T23:58:15,792 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:15,801 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T23:58:15,802 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:15,802 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for a41ba930d231212908b11d93742f8c4a: Waiting for close lock at 1733788695792Running coprocessor pre-close hooks at 1733788695792Disabling compacts and flushes for region at 1733788695792Disabling writes for close at 1733788695792Writing region close event to WAL at 1733788695793 (+1 ms)Running coprocessor post-close hooks at 1733788695802 (+9 ms)Closed at 1733788695802 2024-12-09T23:58:15,802 WARN [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionServer(3299): Not adding moved region record: a41ba930d231212908b11d93742f8c4a to self. 2024-12-09T23:58:15,804 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:15,804 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=a41ba930d231212908b11d93742f8c4a, regionState=CLOSED 2024-12-09T23:58:15,808 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 because future has completed 2024-12-09T23:58:15,810 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-12-09T23:58:15,811 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 in 171 msec 2024-12-09T23:58:15,811 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a41ba930d231212908b11d93742f8c4a, REOPEN/MOVE; state=CLOSED, location=2907d75fbb3e,37627,1733788599837; forceNewPlan=false, retain=true 2024-12-09T23:58:15,962 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=a41ba930d231212908b11d93742f8c4a, regionState=OPENING, regionLocation=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:15,965 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a41ba930d231212908b11d93742f8c4a, REOPEN/MOVE because future has completed 2024-12-09T23:58:15,966 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837}] 2024-12-09T23:58:16,126 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:16,126 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7752): Opening region: {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} 2024-12-09T23:58:16,127 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:16,127 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:58:16,127 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7794): checking encryption for a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:16,127 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7797): checking classloading for a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:16,129 INFO [StoreOpener-a41ba930d231212908b11d93742f8c4a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:16,131 INFO [StoreOpener-a41ba930d231212908b11d93742f8c4a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T23:58:16,131 INFO [StoreOpener-a41ba930d231212908b11d93742f8c4a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a41ba930d231212908b11d93742f8c4a columnFamilyName A 2024-12-09T23:58:16,133 DEBUG [StoreOpener-a41ba930d231212908b11d93742f8c4a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:58:16,134 INFO [StoreOpener-a41ba930d231212908b11d93742f8c4a-1 {}] regionserver.HStore(327): Store=a41ba930d231212908b11d93742f8c4a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:58:16,134 INFO [StoreOpener-a41ba930d231212908b11d93742f8c4a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:16,135 INFO [StoreOpener-a41ba930d231212908b11d93742f8c4a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T23:58:16,135 INFO [StoreOpener-a41ba930d231212908b11d93742f8c4a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a41ba930d231212908b11d93742f8c4a columnFamilyName B 2024-12-09T23:58:16,135 DEBUG [StoreOpener-a41ba930d231212908b11d93742f8c4a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:58:16,136 INFO [StoreOpener-a41ba930d231212908b11d93742f8c4a-1 {}] regionserver.HStore(327): Store=a41ba930d231212908b11d93742f8c4a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:58:16,136 INFO [StoreOpener-a41ba930d231212908b11d93742f8c4a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:16,137 INFO [StoreOpener-a41ba930d231212908b11d93742f8c4a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T23:58:16,137 INFO [StoreOpener-a41ba930d231212908b11d93742f8c4a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a41ba930d231212908b11d93742f8c4a columnFamilyName C 2024-12-09T23:58:16,138 DEBUG [StoreOpener-a41ba930d231212908b11d93742f8c4a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:58:16,138 INFO [StoreOpener-a41ba930d231212908b11d93742f8c4a-1 {}] regionserver.HStore(327): Store=a41ba930d231212908b11d93742f8c4a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:58:16,138 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:16,139 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1038): replaying wal for a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:16,139 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:16,140 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:16,141 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1048): stopping wal replay for a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:16,141 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1060): Cleaning up temporary data for a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:16,141 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T23:58:16,142 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1093): writing seq id for a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:16,143 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1114): Opened a41ba930d231212908b11d93742f8c4a; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71848806, jitterRate=0.07063063979148865}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T23:58:16,143 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:16,143 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1006): Region open journal for a41ba930d231212908b11d93742f8c4a: Running coprocessor pre-open hook at 1733788696127Writing region info on filesystem at 1733788696127Initializing all the Stores at 1733788696129 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788696129Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788696129Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788696129Cleaning up temporary data from old regions at 1733788696141 (+12 ms)Running coprocessor post-open hooks at 1733788696143 (+2 ms)Region opened successfully at 1733788696143 2024-12-09T23:58:16,144 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., pid=90, masterSystemTime=1733788696119 2024-12-09T23:58:16,146 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:16,146 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:16,146 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=a41ba930d231212908b11d93742f8c4a, regionState=OPEN, openSeqNum=5, regionLocation=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:16,147 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 because future has completed 2024-12-09T23:58:16,149 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=88 2024-12-09T23:58:16,149 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 in 183 msec 2024-12-09T23:58:16,150 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-12-09T23:58:16,150 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a41ba930d231212908b11d93742f8c4a, REOPEN/MOVE in 514 msec 2024-12-09T23:58:16,152 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-09T23:58:16,152 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 518 msec 2024-12-09T23:58:16,154 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 942 msec 2024-12-09T23:58:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-09T23:58:16,155 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:58:16,156 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad3f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:16,156 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:58:16,156 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:58:16,157 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:58:16,157 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:58:16,157 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:58:16,157 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da5936, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:16,157 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:58:16,157 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:58:16,157 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:16,158 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57990, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:58:16,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c3e72cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:16,159 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422fcba0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:16,159 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:58:16,159 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:58:16,160 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:58:16,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:58:16,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:58:16,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70f7e2be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:16,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:58:16,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:58:16,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:16,161 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58014, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:58:16,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@da4d6a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:16,162 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72e71a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:16,162 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:58:16,162 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:58:16,162 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:58:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:58:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:58:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ba1ca0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:58:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:58:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:16,164 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58034, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:58:16,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78e05220, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:16,165 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b9dbe2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:16,165 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:58:16,165 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:58:16,166 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:58:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:58:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:58:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45d92974, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:58:16,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:58:16,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:16,167 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:58:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15796e61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:16,168 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eba3e2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:16,168 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:58:16,169 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:58:16,169 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:58:16,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:58:16,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:58:16,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56c3a030, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:16,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:58:16,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:58:16,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:16,171 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58064, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:58:16,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62875376, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:16,172 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d41a701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:16,172 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:58:16,172 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:58:16,173 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:58:16,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:58:16,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:58:16,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36e7af35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:16,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:58:16,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:58:16,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:16,174 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:58:16,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45bfaaa7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:16,176 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2f46ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:16,176 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:58:16,176 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:58:16,177 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:58:16,177 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:58:16,177 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:58:16,177 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3516609b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:16,177 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:58:16,177 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:58:16,177 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:16,178 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58100, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:58:16,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77e5eade, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:16,179 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180bd544, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:16,179 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:58:16,179 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:58:16,180 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:58:16,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:58:16,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:58:16,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@97f62b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:16,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:58:16,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:58:16,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:16,181 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58126, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:58:16,182 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e3c3cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:16,182 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d553196, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:16,182 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:58:16,182 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:58:16,183 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:58:16,183 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:58:16,183 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:58:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@701c2100, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:58:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:58:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:16,185 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58148, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:58:16,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514e01aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:16,186 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dc40216, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:16,186 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:58:16,186 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:58:16,187 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:58:16,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:58:16,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:58:16,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@168e5cc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:16,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:58:16,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:58:16,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:16,188 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:58:16,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1159ca35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:16,189 DEBUG [Thread-1397 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:58:16,189 DEBUG [Thread-1399 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:58:16,190 DEBUG [Thread-1395 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:58:16,190 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:58:16,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:58:16,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:58:16,190 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:58:16,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:58:16,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:58:16,191 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:58:16,192 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57100, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:58:16,192 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57116, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:58:16,192 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57120, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:58:16,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-12-09T23:58:16,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-09T23:58:16,193 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:58:16,194 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:58:16,194 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:58:16,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5] 2024-12-09T23:58:16,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5] 2024-12-09T23:58:16,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5] 2024-12-09T23:58:16,199 DEBUG [Thread-1401 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:58:16,199 DEBUG [Thread-1390 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:58:16,199 DEBUG [Thread-1403 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:58:16,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:58:16,200 DEBUG [Thread-1390 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:58:16,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:58:16,200 DEBUG [Thread-1384 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:58:16,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:58:16,200 DEBUG [Thread-1390 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:58:16,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:58:16,201 DEBUG [Thread-1392 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:58:16,201 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:58:16,201 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:58:16,201 DEBUG [Thread-1386 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:58:16,201 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57146, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:58:16,201 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57136, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:58:16,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:58:16,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:58:16,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:58:16,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:58:16,202 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57164, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:58:16,203 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57162, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:58:16,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5] 2024-12-09T23:58:16,204 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57188, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:58:16,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5] 2024-12-09T23:58:16,205 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57186, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:58:16,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5] 2024-12-09T23:58:16,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5] 2024-12-09T23:58:16,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5] 2024-12-09T23:58:16,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5] 2024-12-09T23:58:16,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:16,214 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a41ba930d231212908b11d93742f8c4a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T23:58:16,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=A 2024-12-09T23:58:16,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:16,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=B 2024-12-09T23:58:16,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:16,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=C 2024-12-09T23:58:16,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:16,215 DEBUG [Thread-1388 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:58:16,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:58:16,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:58:16,217 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57206, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:58:16,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5] 2024-12-09T23:58:16,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d4d5cd7c8b3d4955b173f42ddb6d2851_a41ba930d231212908b11d93742f8c4a is 50, key is test_row_0/A:col10/1733788696214/Put/seqid=0 2024-12-09T23:58:16,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:16,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788756243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:16,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:16,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788756244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:16,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788756245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:16,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:16,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788756247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:16,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:16,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:16,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:16,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788756249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:16,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:16,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742124_1300 (size=12154) 2024-12-09T23:58:16,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-09T23:58:16,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-09T23:58:16,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:16,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:16,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:16,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:16,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:16,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:16,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:16,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788756362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:16,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:16,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:16,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788756363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:16,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:16,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:16,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788756363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:16,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:16,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:16,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788756363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:16,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:16,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:16,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788756363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:16,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:16,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-09T23:58:16,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-09T23:58:16,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:16,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:16,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:16,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:16,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:16,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:16,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:16,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788756573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:16,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:16,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:16,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788756573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:16,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:16,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788756573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:16,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:16,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788756573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:16,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:16,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788756573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:16,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:16,660 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:58:16,664 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d4d5cd7c8b3d4955b173f42ddb6d2851_a41ba930d231212908b11d93742f8c4a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d4d5cd7c8b3d4955b173f42ddb6d2851_a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:16,665 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/113ccf98314b40feb5fc031a77d349d0, store: [table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:16,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/113ccf98314b40feb5fc031a77d349d0 is 175, key is test_row_0/A:col10/1733788696214/Put/seqid=0 2024-12-09T23:58:16,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742125_1301 (size=30955) 2024-12-09T23:58:16,669 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/113ccf98314b40feb5fc031a77d349d0 2024-12-09T23:58:16,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/657d57f7b27c440ca69fec4f185c98ac is 50, key is test_row_0/B:col10/1733788696214/Put/seqid=0 2024-12-09T23:58:16,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742126_1302 (size=12001) 2024-12-09T23:58:16,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/657d57f7b27c440ca69fec4f185c98ac 2024-12-09T23:58:16,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-09T23:58:16,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:16,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:16,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:16,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:16,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:16,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:16,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/9c5369322ec34e07bc9c4dc8d7fa4f48 is 50, key is test_row_0/C:col10/1733788696214/Put/seqid=0 2024-12-09T23:58:16,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742127_1303 (size=12001) 2024-12-09T23:58:16,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/9c5369322ec34e07bc9c4dc8d7fa4f48 2024-12-09T23:58:16,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/113ccf98314b40feb5fc031a77d349d0 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/113ccf98314b40feb5fc031a77d349d0 2024-12-09T23:58:16,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/113ccf98314b40feb5fc031a77d349d0, entries=150, sequenceid=16, filesize=30.2 K 2024-12-09T23:58:16,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/657d57f7b27c440ca69fec4f185c98ac as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/657d57f7b27c440ca69fec4f185c98ac 2024-12-09T23:58:16,756 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/657d57f7b27c440ca69fec4f185c98ac, entries=150, sequenceid=16, filesize=11.7 K 2024-12-09T23:58:16,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/9c5369322ec34e07bc9c4dc8d7fa4f48 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/9c5369322ec34e07bc9c4dc8d7fa4f48 2024-12-09T23:58:16,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/9c5369322ec34e07bc9c4dc8d7fa4f48, entries=150, sequenceid=16, filesize=11.7 K 2024-12-09T23:58:16,764 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a41ba930d231212908b11d93742f8c4a in 550ms, sequenceid=16, compaction requested=false 2024-12-09T23:58:16,764 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-09T23:58:16,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a41ba930d231212908b11d93742f8c4a: 2024-12-09T23:58:16,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-09T23:58:16,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-09T23:58:16,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:16,853 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing a41ba930d231212908b11d93742f8c4a 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T23:58:16,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=A 2024-12-09T23:58:16,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:16,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=B 2024-12-09T23:58:16,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:16,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=C 2024-12-09T23:58:16,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:16,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209378611ce5a8d47c5a2940c86f302271f_a41ba930d231212908b11d93742f8c4a is 50, key is test_row_0/A:col10/1733788696245/Put/seqid=0 2024-12-09T23:58:16,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:16,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:16,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742128_1304 (size=12154) 2024-12-09T23:58:16,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:58:16,893 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209378611ce5a8d47c5a2940c86f302271f_a41ba930d231212908b11d93742f8c4a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209378611ce5a8d47c5a2940c86f302271f_a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:16,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/e04d38025de341159020218d51fae15d, store: [table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:16,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/e04d38025de341159020218d51fae15d is 175, key is test_row_0/A:col10/1733788696245/Put/seqid=0 2024-12-09T23:58:16,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:16,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788756891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:16,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742129_1305 (size=30955) 2024-12-09T23:58:16,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:16,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:16,899 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/e04d38025de341159020218d51fae15d 2024-12-09T23:58:16,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788756892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:16,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:16,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:16,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788756894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:16,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:16,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/db8290db3c524ff1b3afb72e6898e6b8 is 50, key is test_row_0/B:col10/1733788696245/Put/seqid=0 2024-12-09T23:58:16,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:16,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788756897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:16,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:16,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788756898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:16,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:16,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:16,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:16,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742130_1306 (size=12001) 2024-12-09T23:58:16,911 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/db8290db3c524ff1b3afb72e6898e6b8 2024-12-09T23:58:16,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/171d5358f36845ceafbf8fb213a60450 is 50, key is test_row_0/C:col10/1733788696245/Put/seqid=0 2024-12-09T23:58:16,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742131_1307 (size=12001) 2024-12-09T23:58:16,923 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/171d5358f36845ceafbf8fb213a60450 2024-12-09T23:58:16,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/e04d38025de341159020218d51fae15d as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/e04d38025de341159020218d51fae15d 2024-12-09T23:58:16,932 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/e04d38025de341159020218d51fae15d, entries=150, sequenceid=36, filesize=30.2 K 2024-12-09T23:58:16,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/db8290db3c524ff1b3afb72e6898e6b8 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/db8290db3c524ff1b3afb72e6898e6b8 2024-12-09T23:58:16,936 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/db8290db3c524ff1b3afb72e6898e6b8, entries=150, sequenceid=36, filesize=11.7 K 2024-12-09T23:58:16,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/171d5358f36845ceafbf8fb213a60450 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/171d5358f36845ceafbf8fb213a60450 2024-12-09T23:58:16,944 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/171d5358f36845ceafbf8fb213a60450, entries=150, sequenceid=36, filesize=11.7 K 2024-12-09T23:58:16,945 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a41ba930d231212908b11d93742f8c4a in 92ms, sequenceid=36, compaction requested=false 2024-12-09T23:58:16,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for a41ba930d231212908b11d93742f8c4a: 2024-12-09T23:58:16,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:16,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-12-09T23:58:16,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-12-09T23:58:16,950 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-12-09T23:58:16,950 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 755 msec 2024-12-09T23:58:16,953 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 760 msec 2024-12-09T23:58:17,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:17,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a41ba930d231212908b11d93742f8c4a 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T23:58:17,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=A 2024-12-09T23:58:17,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:17,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=B 2024-12-09T23:58:17,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:17,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=C 2024-12-09T23:58:17,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:17,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209040435876ade4e1785f1c0bffc3bfcee_a41ba930d231212908b11d93742f8c4a is 50, key is test_row_0/A:col10/1733788696886/Put/seqid=0 2024-12-09T23:58:17,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742132_1308 (size=17034) 2024-12-09T23:58:17,031 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:58:17,036 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209040435876ade4e1785f1c0bffc3bfcee_a41ba930d231212908b11d93742f8c4a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209040435876ade4e1785f1c0bffc3bfcee_a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:17,037 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/1c3cc55643874b41b4b0119cb7273d05, store: [table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:17,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/1c3cc55643874b41b4b0119cb7273d05 is 175, key is test_row_0/A:col10/1733788696886/Put/seqid=0 2024-12-09T23:58:17,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:17,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788757030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:17,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:17,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:17,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788757032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:17,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:17,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788757036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:17,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:17,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:17,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:17,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788757037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:17,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:17,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788757038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:17,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:17,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:17,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742133_1309 (size=48139) 2024-12-09T23:58:17,053 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/1c3cc55643874b41b4b0119cb7273d05 2024-12-09T23:58:17,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/d60aedb81d014d159566b200026308c4 is 50, key is test_row_0/B:col10/1733788696886/Put/seqid=0 2024-12-09T23:58:17,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742134_1310 (size=12001) 2024-12-09T23:58:17,073 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/d60aedb81d014d159566b200026308c4 2024-12-09T23:58:17,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/d5833ebcdb9f4cc696c71bfd7898eed8 is 50, key is test_row_0/C:col10/1733788696886/Put/seqid=0 2024-12-09T23:58:17,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742135_1311 (size=12001) 2024-12-09T23:58:17,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:17,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788757142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:17,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:17,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:17,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788757153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:17,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:17,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:17,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788757153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:17,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:17,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:17,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788757153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:17,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:17,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:17,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788757153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:17,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:17,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-09T23:58:17,333 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:58:17,333 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:58:17,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-12-09T23:58:17,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-09T23:58:17,335 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:58:17,335 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:58:17,336 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:58:17,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:17,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788757363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:17,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:17,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788757363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:17,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:17,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:17,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788757363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:17,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:17,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:17,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788757363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:17,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:17,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:17,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788757363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:17,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:17,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:17,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-09T23:58:17,485 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/d5833ebcdb9f4cc696c71bfd7898eed8 2024-12-09T23:58:17,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-09T23:58:17,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:17,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:17,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:17,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:17,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:17,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/1c3cc55643874b41b4b0119cb7273d05 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/1c3cc55643874b41b4b0119cb7273d05 2024-12-09T23:58:17,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:17,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/1c3cc55643874b41b4b0119cb7273d05, entries=250, sequenceid=50, filesize=47.0 K 2024-12-09T23:58:17,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/d60aedb81d014d159566b200026308c4 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/d60aedb81d014d159566b200026308c4 2024-12-09T23:58:17,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/d60aedb81d014d159566b200026308c4, entries=150, sequenceid=50, filesize=11.7 K 2024-12-09T23:58:17,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/d5833ebcdb9f4cc696c71bfd7898eed8 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/d5833ebcdb9f4cc696c71bfd7898eed8 2024-12-09T23:58:17,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/d5833ebcdb9f4cc696c71bfd7898eed8, entries=150, sequenceid=50, filesize=11.7 K 2024-12-09T23:58:17,501 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for a41ba930d231212908b11d93742f8c4a in 495ms, sequenceid=50, compaction requested=true 2024-12-09T23:58:17,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a41ba930d231212908b11d93742f8c4a: 2024-12-09T23:58:17,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a41ba930d231212908b11d93742f8c4a:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:58:17,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:17,502 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:58:17,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a41ba930d231212908b11d93742f8c4a:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:58:17,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:17,502 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:58:17,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a41ba930d231212908b11d93742f8c4a:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:58:17,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:58:17,502 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:58:17,502 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:58:17,503 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): a41ba930d231212908b11d93742f8c4a/B is initiating minor compaction (all files) 2024-12-09T23:58:17,503 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): a41ba930d231212908b11d93742f8c4a/A is initiating minor compaction (all files) 2024-12-09T23:58:17,503 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a41ba930d231212908b11d93742f8c4a/A in TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:17,503 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a41ba930d231212908b11d93742f8c4a/B in TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:17,503 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/113ccf98314b40feb5fc031a77d349d0, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/e04d38025de341159020218d51fae15d, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/1c3cc55643874b41b4b0119cb7273d05] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp, totalSize=107.5 K 2024-12-09T23:58:17,503 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/657d57f7b27c440ca69fec4f185c98ac, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/db8290db3c524ff1b3afb72e6898e6b8, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/d60aedb81d014d159566b200026308c4] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp, totalSize=35.2 K 2024-12-09T23:58:17,503 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:17,503 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. files: [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/113ccf98314b40feb5fc031a77d349d0, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/e04d38025de341159020218d51fae15d, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/1c3cc55643874b41b4b0119cb7273d05] 2024-12-09T23:58:17,503 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 657d57f7b27c440ca69fec4f185c98ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733788696205 2024-12-09T23:58:17,503 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 113ccf98314b40feb5fc031a77d349d0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733788696205 2024-12-09T23:58:17,503 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting db8290db3c524ff1b3afb72e6898e6b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733788696239 2024-12-09T23:58:17,503 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting e04d38025de341159020218d51fae15d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733788696239 2024-12-09T23:58:17,504 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting d60aedb81d014d159566b200026308c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733788696886 2024-12-09T23:58:17,504 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c3cc55643874b41b4b0119cb7273d05, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733788696886 2024-12-09T23:58:17,509 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:17,513 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a41ba930d231212908b11d93742f8c4a#B#compaction#268 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:17,513 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/9f84e5c505754ca4b34c6463ff9446e9 is 50, key is test_row_0/B:col10/1733788696886/Put/seqid=0 2024-12-09T23:58:17,514 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412095475da8a6a514f36a9a612bb6772628a_a41ba930d231212908b11d93742f8c4a store=[table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:17,516 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412095475da8a6a514f36a9a612bb6772628a_a41ba930d231212908b11d93742f8c4a, store=[table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:17,516 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412095475da8a6a514f36a9a612bb6772628a_a41ba930d231212908b11d93742f8c4a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:17,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742136_1312 (size=12104) 2024-12-09T23:58:17,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742137_1313 (size=4469) 2024-12-09T23:58:17,528 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a41ba930d231212908b11d93742f8c4a#A#compaction#267 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:17,529 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/a59d70e4ae6e45339889b8bd957d2ff0 is 175, key is test_row_0/A:col10/1733788696886/Put/seqid=0 2024-12-09T23:58:17,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742138_1314 (size=31058) 2024-12-09T23:58:17,538 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/a59d70e4ae6e45339889b8bd957d2ff0 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/a59d70e4ae6e45339889b8bd957d2ff0 2024-12-09T23:58:17,542 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a41ba930d231212908b11d93742f8c4a/A of a41ba930d231212908b11d93742f8c4a into a59d70e4ae6e45339889b8bd957d2ff0(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:17,542 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a41ba930d231212908b11d93742f8c4a: 2024-12-09T23:58:17,542 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., storeName=a41ba930d231212908b11d93742f8c4a/A, priority=13, startTime=1733788697502; duration=0sec 2024-12-09T23:58:17,542 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:58:17,542 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a41ba930d231212908b11d93742f8c4a:A 2024-12-09T23:58:17,542 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:58:17,543 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:58:17,543 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): a41ba930d231212908b11d93742f8c4a/C is initiating minor compaction (all files) 2024-12-09T23:58:17,543 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a41ba930d231212908b11d93742f8c4a/C in TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:17,543 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/9c5369322ec34e07bc9c4dc8d7fa4f48, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/171d5358f36845ceafbf8fb213a60450, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/d5833ebcdb9f4cc696c71bfd7898eed8] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp, totalSize=35.2 K 2024-12-09T23:58:17,543 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c5369322ec34e07bc9c4dc8d7fa4f48, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733788696205 2024-12-09T23:58:17,544 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 171d5358f36845ceafbf8fb213a60450, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733788696239 2024-12-09T23:58:17,544 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5833ebcdb9f4cc696c71bfd7898eed8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733788696886 2024-12-09T23:58:17,549 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a41ba930d231212908b11d93742f8c4a#C#compaction#269 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:17,549 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/23401302d1214fe894c887a436c15f21 is 50, key is test_row_0/C:col10/1733788696886/Put/seqid=0 2024-12-09T23:58:17,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742139_1315 (size=12104) 2024-12-09T23:58:17,558 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/23401302d1214fe894c887a436c15f21 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/23401302d1214fe894c887a436c15f21 2024-12-09T23:58:17,564 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a41ba930d231212908b11d93742f8c4a/C of a41ba930d231212908b11d93742f8c4a into 23401302d1214fe894c887a436c15f21(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:17,564 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a41ba930d231212908b11d93742f8c4a: 2024-12-09T23:58:17,564 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., storeName=a41ba930d231212908b11d93742f8c4a/C, priority=13, startTime=1733788697502; duration=0sec 2024-12-09T23:58:17,564 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:17,564 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a41ba930d231212908b11d93742f8c4a:C 2024-12-09T23:58:17,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-09T23:58:17,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:17,641 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing a41ba930d231212908b11d93742f8c4a 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-09T23:58:17,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=A 2024-12-09T23:58:17,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:17,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=B 2024-12-09T23:58:17,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:17,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=C 2024-12-09T23:58:17,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:17,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412095f8d08ef2d994eca98dc0e660bdd2dd4_a41ba930d231212908b11d93742f8c4a is 50, key is test_row_0/A:col10/1733788697028/Put/seqid=0 2024-12-09T23:58:17,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742140_1316 (size=12154) 2024-12-09T23:58:17,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:58:17,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-09T23:58:17,656 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412095f8d08ef2d994eca98dc0e660bdd2dd4_a41ba930d231212908b11d93742f8c4a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412095f8d08ef2d994eca98dc0e660bdd2dd4_a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:17,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/9106e0fb57a54898a2c362c788e1493f, store: [table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:17,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/9106e0fb57a54898a2c362c788e1493f is 175, key is test_row_0/A:col10/1733788697028/Put/seqid=0 2024-12-09T23:58:17,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742141_1317 (size=30955) 2024-12-09T23:58:17,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:17,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:17,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:17,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788757686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:17,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:17,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:17,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788757688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:17,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:17,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788757688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:17,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:17,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:17,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:17,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788757688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:17,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:17,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:17,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788757689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:17,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:17,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:17,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788757803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:17,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:17,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:17,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788757803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:17,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788757803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:17,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788757803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:17,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788757803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:17,845 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T23:58:17,923 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/9f84e5c505754ca4b34c6463ff9446e9 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/9f84e5c505754ca4b34c6463ff9446e9 2024-12-09T23:58:17,926 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a41ba930d231212908b11d93742f8c4a/B of a41ba930d231212908b11d93742f8c4a into 9f84e5c505754ca4b34c6463ff9446e9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:17,926 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a41ba930d231212908b11d93742f8c4a: 2024-12-09T23:58:17,926 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., storeName=a41ba930d231212908b11d93742f8c4a/B, priority=13, startTime=1733788697502; duration=0sec 2024-12-09T23:58:17,926 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:17,927 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a41ba930d231212908b11d93742f8c4a:B 2024-12-09T23:58:17,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-09T23:58:18,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:18,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788758013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:18,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:18,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:18,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:18,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788758013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:18,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:18,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:18,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:18,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788758013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:18,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:18,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:18,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:18,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:18,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788758013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:18,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:18,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:18,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:18,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:18,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:18,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788758014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:18,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:18,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:18,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:18,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:18,061 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=70, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/9106e0fb57a54898a2c362c788e1493f 2024-12-09T23:58:18,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/82fd67353853480b840cfa1fb1623cf1 is 50, key is test_row_0/B:col10/1733788697028/Put/seqid=0 2024-12-09T23:58:18,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742142_1318 (size=12001) 2024-12-09T23:58:18,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788758323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:18,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:18,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788758323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:18,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:18,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:18,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:18,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788758323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:18,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:18,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:18,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:18,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788758323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:18,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:18,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:18,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:18,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:18,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:18,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788758323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:18,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:18,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:18,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:18,470 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/82fd67353853480b840cfa1fb1623cf1 2024-12-09T23:58:18,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-09T23:58:18,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/c299cd83d09b4228934170feca6bc897 is 50, key is test_row_0/C:col10/1733788697028/Put/seqid=0 2024-12-09T23:58:18,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742143_1319 (size=12001) 2024-12-09T23:58:18,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788758832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:18,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:18,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788758832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:18,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:18,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788758833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:18,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:18,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788758833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:18,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:18,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788758833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:18,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:18,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:18,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:18,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:18,882 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/c299cd83d09b4228934170feca6bc897 2024-12-09T23:58:18,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/9106e0fb57a54898a2c362c788e1493f as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/9106e0fb57a54898a2c362c788e1493f 2024-12-09T23:58:18,890 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/9106e0fb57a54898a2c362c788e1493f, entries=150, sequenceid=70, filesize=30.2 K 2024-12-09T23:58:18,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/82fd67353853480b840cfa1fb1623cf1 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/82fd67353853480b840cfa1fb1623cf1 2024-12-09T23:58:18,894 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/82fd67353853480b840cfa1fb1623cf1, entries=150, sequenceid=70, filesize=11.7 K 2024-12-09T23:58:18,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/c299cd83d09b4228934170feca6bc897 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/c299cd83d09b4228934170feca6bc897 2024-12-09T23:58:18,898 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/c299cd83d09b4228934170feca6bc897, entries=150, sequenceid=70, filesize=11.7 K 2024-12-09T23:58:18,899 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for a41ba930d231212908b11d93742f8c4a in 1258ms, sequenceid=70, compaction requested=false 2024-12-09T23:58:18,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for a41ba930d231212908b11d93742f8c4a: 2024-12-09T23:58:18,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:18,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-12-09T23:58:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-12-09T23:58:18,902 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-09T23:58:18,903 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5650 sec 2024-12-09T23:58:18,905 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 1.5700 sec 2024-12-09T23:58:19,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-09T23:58:19,483 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:58:19,483 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:58:19,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-12-09T23:58:19,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-09T23:58:19,485 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:58:19,486 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:58:19,486 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:58:19,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-09T23:58:19,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-09T23:58:19,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:19,638 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing a41ba930d231212908b11d93742f8c4a 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-09T23:58:19,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=A 2024-12-09T23:58:19,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:19,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=B 2024-12-09T23:58:19,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:19,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=C 2024-12-09T23:58:19,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:19,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120911442a420a28452f9db0c3c1caed928e_a41ba930d231212908b11d93742f8c4a is 50, key is test_row_0/A:col10/1733788697685/Put/seqid=0 2024-12-09T23:58:19,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742144_1320 (size=12154) 2024-12-09T23:58:19,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-09T23:58:19,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:19,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:19,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:19,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788759870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:19,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:19,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:19,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:19,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:19,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788759876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:19,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:19,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:19,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:19,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:19,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788759877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:19,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:19,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788759878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:19,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:19,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788759878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:19,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:19,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:19,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:19,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:19,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788759983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:19,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:19,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:19,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:19,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:19,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788759993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:19,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:19,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:19,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:19,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:19,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788759993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:19,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:19,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:19,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:19,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:19,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788759993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:19,997 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:19,997 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:19,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:19,997 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:19,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788759993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:19,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:19,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:19,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:20,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:58:20,054 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120911442a420a28452f9db0c3c1caed928e_a41ba930d231212908b11d93742f8c4a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120911442a420a28452f9db0c3c1caed928e_a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:20,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/1d78d53a064649cc8028e98f44bd2e1c, store: [table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:20,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/1d78d53a064649cc8028e98f44bd2e1c is 175, key is test_row_0/A:col10/1733788697685/Put/seqid=0 2024-12-09T23:58:20,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742145_1321 (size=30955) 2024-12-09T23:58:20,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-09T23:58:20,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:20,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788760193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:20,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:20,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:20,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788760202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:20,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:20,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788760202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:20,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:20,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:20,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:20,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788760202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:20,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:20,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788760203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:20,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:20,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:20,459 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=85, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/1d78d53a064649cc8028e98f44bd2e1c 2024-12-09T23:58:20,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/f7f1b6057e3148a8b4884860647d5e69 is 50, key is test_row_0/B:col10/1733788697685/Put/seqid=0 2024-12-09T23:58:20,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742146_1322 (size=12001) 2024-12-09T23:58:20,471 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/f7f1b6057e3148a8b4884860647d5e69 2024-12-09T23:58:20,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/c6530ec49e1d4ad595bfb9f16e3d096c is 50, key is test_row_0/C:col10/1733788697685/Put/seqid=0 2024-12-09T23:58:20,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742147_1323 (size=12001) 2024-12-09T23:58:20,484 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/c6530ec49e1d4ad595bfb9f16e3d096c 2024-12-09T23:58:20,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/1d78d53a064649cc8028e98f44bd2e1c as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/1d78d53a064649cc8028e98f44bd2e1c 2024-12-09T23:58:20,491 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/1d78d53a064649cc8028e98f44bd2e1c, entries=150, sequenceid=85, filesize=30.2 K 2024-12-09T23:58:20,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/f7f1b6057e3148a8b4884860647d5e69 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/f7f1b6057e3148a8b4884860647d5e69 2024-12-09T23:58:20,497 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/f7f1b6057e3148a8b4884860647d5e69, entries=150, sequenceid=85, filesize=11.7 K 2024-12-09T23:58:20,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/c6530ec49e1d4ad595bfb9f16e3d096c as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/c6530ec49e1d4ad595bfb9f16e3d096c 2024-12-09T23:58:20,502 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/c6530ec49e1d4ad595bfb9f16e3d096c, entries=150, sequenceid=85, filesize=11.7 K 2024-12-09T23:58:20,502 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for a41ba930d231212908b11d93742f8c4a in 864ms, sequenceid=85, compaction requested=true 2024-12-09T23:58:20,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for a41ba930d231212908b11d93742f8c4a: 2024-12-09T23:58:20,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:20,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-09T23:58:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-12-09T23:58:20,505 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-09T23:58:20,505 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0180 sec 2024-12-09T23:58:20,506 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.0230 sec 2024-12-09T23:58:20,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:20,508 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a41ba930d231212908b11d93742f8c4a 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T23:58:20,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=A 2024-12-09T23:58:20,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:20,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=B 2024-12-09T23:58:20,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:20,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=C 2024-12-09T23:58:20,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:20,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209a15ad80f563142479033c79e9e066670_a41ba930d231212908b11d93742f8c4a is 50, key is test_row_0/A:col10/1733788700507/Put/seqid=0 2024-12-09T23:58:20,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742148_1324 (size=14594) 2024-12-09T23:58:20,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:20,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788760525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:20,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:20,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:20,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788760534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:20,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:20,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:20,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788760534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:20,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:20,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:20,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788760534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:20,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:20,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:20,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788760534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:20,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:20,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-09T23:58:20,623 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:58:20,623 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:58:20,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-09T23:58:20,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T23:58:20,624 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:58:20,625 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:58:20,625 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:58:20,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:20,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788760643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:20,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:20,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:20,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788760653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:20,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:20,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788760653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:20,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:20,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:20,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788760653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:20,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:20,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:20,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788760653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:20,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:20,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:20,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T23:58:20,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T23:58:20,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:20,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:20,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:20,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:20,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:20,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:20,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:20,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788760853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:20,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:20,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:20,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788760862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:20,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:20,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:20,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788760872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:20,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:20,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788760872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:20,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:20,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:20,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:20,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788760873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:20,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:20,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:20,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T23:58:20,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:20,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:20,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:20,930 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:58:20,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:20,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:20,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:20,933 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209a15ad80f563142479033c79e9e066670_a41ba930d231212908b11d93742f8c4a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a15ad80f563142479033c79e9e066670_a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:20,934 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/0a00ecb419404fdd89b2f8fbcc51f5f6, store: [table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:20,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/0a00ecb419404fdd89b2f8fbcc51f5f6 is 175, key is test_row_0/A:col10/1733788700507/Put/seqid=0 2024-12-09T23:58:20,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742149_1325 (size=39549) 2024-12-09T23:58:20,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T23:58:21,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T23:58:21,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:21,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:21,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:21,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:21,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:21,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:21,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:21,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788761163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:21,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:21,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:21,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:21,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:21,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788761173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:21,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:21,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:21,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:21,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:21,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788761182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:21,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:21,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:21,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:21,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788761182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:21,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:21,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:21,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:21,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:21,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:21,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788761182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:21,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:21,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:21,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:21,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T23:58:21,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:21,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:21,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:21,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:21,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:21,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:21,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T23:58:21,339 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=104, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/0a00ecb419404fdd89b2f8fbcc51f5f6 2024-12-09T23:58:21,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/b6e7fbd7484f42b287024c4d378d9f0f is 50, key is test_row_0/B:col10/1733788700507/Put/seqid=0 2024-12-09T23:58:21,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T23:58:21,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:21,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:21,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:21,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:21,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:21,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:21,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742150_1326 (size=12001) 2024-12-09T23:58:21,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T23:58:21,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:21,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:21,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:21,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:21,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:21,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:21,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:21,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788761672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:21,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:21,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:21,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:21,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:21,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788761692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:21,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:21,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:21,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:21,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:21,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788761693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:21,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:21,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:21,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:21,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:21,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788761693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:21,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:21,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:21,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788761693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:21,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:21,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:21,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:21,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:21,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:21,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T23:58:21,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:21,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:21,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:21,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:21,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:21,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:21,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T23:58:21,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/b6e7fbd7484f42b287024c4d378d9f0f 2024-12-09T23:58:21,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/cc8cdda963204f42a6c7b3444f6adf19 is 50, key is test_row_0/C:col10/1733788700507/Put/seqid=0 2024-12-09T23:58:21,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742151_1327 (size=12001) 2024-12-09T23:58:21,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/cc8cdda963204f42a6c7b3444f6adf19 2024-12-09T23:58:21,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/0a00ecb419404fdd89b2f8fbcc51f5f6 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/0a00ecb419404fdd89b2f8fbcc51f5f6 2024-12-09T23:58:21,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/0a00ecb419404fdd89b2f8fbcc51f5f6, entries=200, sequenceid=104, filesize=38.6 K 2024-12-09T23:58:21,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/b6e7fbd7484f42b287024c4d378d9f0f as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/b6e7fbd7484f42b287024c4d378d9f0f 2024-12-09T23:58:21,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/b6e7fbd7484f42b287024c4d378d9f0f, entries=150, sequenceid=104, filesize=11.7 K 2024-12-09T23:58:21,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/cc8cdda963204f42a6c7b3444f6adf19 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/cc8cdda963204f42a6c7b3444f6adf19 2024-12-09T23:58:21,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/cc8cdda963204f42a6c7b3444f6adf19, entries=150, sequenceid=104, filesize=11.7 K 2024-12-09T23:58:21,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for a41ba930d231212908b11d93742f8c4a in 1321ms, sequenceid=104, compaction requested=true 2024-12-09T23:58:21,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a41ba930d231212908b11d93742f8c4a: 2024-12-09T23:58:21,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a41ba930d231212908b11d93742f8c4a:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:58:21,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:21,829 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T23:58:21,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a41ba930d231212908b11d93742f8c4a:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:58:21,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:21,829 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T23:58:21,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a41ba930d231212908b11d93742f8c4a:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:58:21,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:58:21,830 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T23:58:21,830 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132517 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T23:58:21,830 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): a41ba930d231212908b11d93742f8c4a/A is initiating minor compaction (all files) 2024-12-09T23:58:21,830 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): a41ba930d231212908b11d93742f8c4a/B is initiating minor compaction (all files) 2024-12-09T23:58:21,830 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a41ba930d231212908b11d93742f8c4a/B in TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:21,830 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a41ba930d231212908b11d93742f8c4a/A in TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:21,830 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/9f84e5c505754ca4b34c6463ff9446e9, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/82fd67353853480b840cfa1fb1623cf1, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/f7f1b6057e3148a8b4884860647d5e69, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/b6e7fbd7484f42b287024c4d378d9f0f] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp, totalSize=47.0 K 2024-12-09T23:58:21,831 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/a59d70e4ae6e45339889b8bd957d2ff0, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/9106e0fb57a54898a2c362c788e1493f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/1d78d53a064649cc8028e98f44bd2e1c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/0a00ecb419404fdd89b2f8fbcc51f5f6] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp, totalSize=129.4 K 2024-12-09T23:58:21,831 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:21,831 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. files: [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/a59d70e4ae6e45339889b8bd957d2ff0, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/9106e0fb57a54898a2c362c788e1493f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/1d78d53a064649cc8028e98f44bd2e1c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/0a00ecb419404fdd89b2f8fbcc51f5f6] 2024-12-09T23:58:21,831 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f84e5c505754ca4b34c6463ff9446e9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733788696886 2024-12-09T23:58:21,831 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting a59d70e4ae6e45339889b8bd957d2ff0, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733788696886 2024-12-09T23:58:21,831 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 82fd67353853480b840cfa1fb1623cf1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1733788697028 2024-12-09T23:58:21,831 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9106e0fb57a54898a2c362c788e1493f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1733788697028 2024-12-09T23:58:21,832 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting f7f1b6057e3148a8b4884860647d5e69, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733788697685 2024-12-09T23:58:21,832 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d78d53a064649cc8028e98f44bd2e1c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733788697685 2024-12-09T23:58:21,832 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting b6e7fbd7484f42b287024c4d378d9f0f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733788699876 2024-12-09T23:58:21,832 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a00ecb419404fdd89b2f8fbcc51f5f6, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733788699868 2024-12-09T23:58:21,838 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:21,841 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a41ba930d231212908b11d93742f8c4a#B#compaction#280 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:21,842 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/9e54be9e153f4fb2ad95f0bc6eb15fca is 50, key is test_row_0/B:col10/1733788700507/Put/seqid=0 2024-12-09T23:58:21,844 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412097f010f937c3148a3977c9c3515b80d10_a41ba930d231212908b11d93742f8c4a store=[table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:21,846 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412097f010f937c3148a3977c9c3515b80d10_a41ba930d231212908b11d93742f8c4a, store=[table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:21,846 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412097f010f937c3148a3977c9c3515b80d10_a41ba930d231212908b11d93742f8c4a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:21,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742152_1328 (size=12241) 2024-12-09T23:58:21,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T23:58:21,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:21,858 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing a41ba930d231212908b11d93742f8c4a 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T23:58:21,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=A 2024-12-09T23:58:21,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:21,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=B 2024-12-09T23:58:21,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:21,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=C 2024-12-09T23:58:21,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:21,862 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/9e54be9e153f4fb2ad95f0bc6eb15fca as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/9e54be9e153f4fb2ad95f0bc6eb15fca 2024-12-09T23:58:21,867 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a41ba930d231212908b11d93742f8c4a/B of a41ba930d231212908b11d93742f8c4a into 9e54be9e153f4fb2ad95f0bc6eb15fca(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:21,867 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a41ba930d231212908b11d93742f8c4a: 2024-12-09T23:58:21,867 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., storeName=a41ba930d231212908b11d93742f8c4a/B, priority=12, startTime=1733788701829; duration=0sec 2024-12-09T23:58:21,869 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:58:21,869 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a41ba930d231212908b11d93742f8c4a:B 2024-12-09T23:58:21,869 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T23:58:21,870 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T23:58:21,870 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): a41ba930d231212908b11d93742f8c4a/C is initiating minor compaction (all files) 2024-12-09T23:58:21,870 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a41ba930d231212908b11d93742f8c4a/C in TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:21,870 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/23401302d1214fe894c887a436c15f21, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/c299cd83d09b4228934170feca6bc897, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/c6530ec49e1d4ad595bfb9f16e3d096c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/cc8cdda963204f42a6c7b3444f6adf19] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp, totalSize=47.0 K 2024-12-09T23:58:21,870 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 23401302d1214fe894c887a436c15f21, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733788696886 2024-12-09T23:58:21,870 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting c299cd83d09b4228934170feca6bc897, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1733788697028 2024-12-09T23:58:21,871 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting c6530ec49e1d4ad595bfb9f16e3d096c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733788697685 2024-12-09T23:58:21,872 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting cc8cdda963204f42a6c7b3444f6adf19, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733788699876 2024-12-09T23:58:21,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742153_1329 (size=4469) 2024-12-09T23:58:21,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ea14a8f17e714226b0b93f49f3f90925_a41ba930d231212908b11d93742f8c4a is 50, key is test_row_0/A:col10/1733788700533/Put/seqid=0 2024-12-09T23:58:21,889 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a41ba930d231212908b11d93742f8c4a#C#compaction#282 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:21,890 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/a2955c9d85ce448dbe0701d6de7c4994 is 50, key is test_row_0/C:col10/1733788700507/Put/seqid=0 2024-12-09T23:58:21,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742154_1330 (size=12154) 2024-12-09T23:58:21,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742155_1331 (size=12241) 2024-12-09T23:58:22,277 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a41ba930d231212908b11d93742f8c4a#A#compaction#279 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:22,277 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/9b489fa75fd54d0da70367020c43f08c is 175, key is test_row_0/A:col10/1733788700507/Put/seqid=0 2024-12-09T23:58:22,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742156_1332 (size=31195) 2024-12-09T23:58:22,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:58:22,310 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ea14a8f17e714226b0b93f49f3f90925_a41ba930d231212908b11d93742f8c4a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ea14a8f17e714226b0b93f49f3f90925_a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:22,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/0cf9439c07204f7b9c8fae1a3d8d00bd, store: [table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:22,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/0cf9439c07204f7b9c8fae1a3d8d00bd is 175, key is test_row_0/A:col10/1733788700533/Put/seqid=0 2024-12-09T23:58:22,312 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/a2955c9d85ce448dbe0701d6de7c4994 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/a2955c9d85ce448dbe0701d6de7c4994 2024-12-09T23:58:22,318 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a41ba930d231212908b11d93742f8c4a/C of a41ba930d231212908b11d93742f8c4a into a2955c9d85ce448dbe0701d6de7c4994(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:22,318 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a41ba930d231212908b11d93742f8c4a: 2024-12-09T23:58:22,318 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., storeName=a41ba930d231212908b11d93742f8c4a/C, priority=12, startTime=1733788701829; duration=0sec 2024-12-09T23:58:22,318 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:22,318 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a41ba930d231212908b11d93742f8c4a:C 2024-12-09T23:58:22,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742157_1333 (size=30955) 2024-12-09T23:58:22,685 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/9b489fa75fd54d0da70367020c43f08c as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/9b489fa75fd54d0da70367020c43f08c 2024-12-09T23:58:22,689 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a41ba930d231212908b11d93742f8c4a/A of a41ba930d231212908b11d93742f8c4a into 9b489fa75fd54d0da70367020c43f08c(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:22,689 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a41ba930d231212908b11d93742f8c4a: 2024-12-09T23:58:22,689 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., storeName=a41ba930d231212908b11d93742f8c4a/A, priority=12, startTime=1733788701829; duration=0sec 2024-12-09T23:58:22,690 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:22,690 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a41ba930d231212908b11d93742f8c4a:A 2024-12-09T23:58:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:22,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:22,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:22,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788762718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:22,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:22,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788762719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:22,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:22,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:22,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:22,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:22,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:22,727 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/0cf9439c07204f7b9c8fae1a3d8d00bd 2024-12-09T23:58:22,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788762721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:22,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:22,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:22,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788762722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:22,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:22,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:22,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:22,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:22,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788762725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:22,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:22,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:22,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:22,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/4c554236fbb04c48a2b2a0de3346915c is 50, key is test_row_0/B:col10/1733788700533/Put/seqid=0 2024-12-09T23:58:22,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742158_1334 (size=12001) 2024-12-09T23:58:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T23:58:22,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788762833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:22,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788762833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:22,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:22,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:22,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:22,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:22,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788762833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:22,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:22,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:22,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:22,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:22,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788762833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:22,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:22,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:22,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:22,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:22,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788762842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:22,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:22,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:22,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:23,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788763043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:23,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788763043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:23,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:23,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:23,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788763043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:23,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:23,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788763043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:23,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:23,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:23,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:23,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788763052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:23,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:23,139 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/4c554236fbb04c48a2b2a0de3346915c 2024-12-09T23:58:23,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/02c6e5e5657f43a9a4c2aaeeca0c688b is 50, key is test_row_0/C:col10/1733788700533/Put/seqid=0 2024-12-09T23:58:23,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742159_1335 (size=12001) 2024-12-09T23:58:23,160 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/02c6e5e5657f43a9a4c2aaeeca0c688b 2024-12-09T23:58:23,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/0cf9439c07204f7b9c8fae1a3d8d00bd as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/0cf9439c07204f7b9c8fae1a3d8d00bd 2024-12-09T23:58:23,167 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/0cf9439c07204f7b9c8fae1a3d8d00bd, entries=150, sequenceid=117, filesize=30.2 K 2024-12-09T23:58:23,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/4c554236fbb04c48a2b2a0de3346915c as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/4c554236fbb04c48a2b2a0de3346915c 2024-12-09T23:58:23,172 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/4c554236fbb04c48a2b2a0de3346915c, entries=150, sequenceid=117, filesize=11.7 K 2024-12-09T23:58:23,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/02c6e5e5657f43a9a4c2aaeeca0c688b as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/02c6e5e5657f43a9a4c2aaeeca0c688b 2024-12-09T23:58:23,177 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/02c6e5e5657f43a9a4c2aaeeca0c688b, entries=150, sequenceid=117, filesize=11.7 K 2024-12-09T23:58:23,178 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for a41ba930d231212908b11d93742f8c4a in 1319ms, sequenceid=117, compaction requested=false 2024-12-09T23:58:23,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for a41ba930d231212908b11d93742f8c4a: 2024-12-09T23:58:23,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:23,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-09T23:58:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-09T23:58:23,181 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-09T23:58:23,181 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5540 sec 2024-12-09T23:58:23,183 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.5580 sec 2024-12-09T23:58:23,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:23,355 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a41ba930d231212908b11d93742f8c4a 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T23:58:23,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=A 2024-12-09T23:58:23,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:23,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=B 2024-12-09T23:58:23,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:23,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=C 2024-12-09T23:58:23,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:23,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209664ccd3a7ee541b5987f5c53a92cd609_a41ba930d231212908b11d93742f8c4a is 50, key is test_row_0/A:col10/1733788702720/Put/seqid=0 2024-12-09T23:58:23,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742160_1336 (size=14794) 2024-12-09T23:58:23,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:23,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788763366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:23,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:23,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788763366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:23,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:23,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:23,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788763367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:23,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:23,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788763369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:23,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:23,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:23,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788763374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:23,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:23,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:23,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788763483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:23,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:23,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788763483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:23,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:23,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:23,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788763483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:23,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:23,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:23,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788763483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:23,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:23,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:23,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788763483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:23,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:23,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:23,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:23,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788763693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:23,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:23,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788763693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:23,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:23,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:23,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:23,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788763693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:23,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:23,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:23,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788763693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:23,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:23,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788763694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:23,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:23,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:23,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:23,765 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:58:23,769 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209664ccd3a7ee541b5987f5c53a92cd609_a41ba930d231212908b11d93742f8c4a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209664ccd3a7ee541b5987f5c53a92cd609_a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:23,770 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/07d5ce375e6e4782ac819d9bc49af226, store: [table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:23,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/07d5ce375e6e4782ac819d9bc49af226 is 175, key is test_row_0/A:col10/1733788702720/Put/seqid=0 2024-12-09T23:58:23,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742161_1337 (size=39749) 2024-12-09T23:58:24,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788764003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:24,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:24,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:24,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788764003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:24,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:24,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:24,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:24,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788764003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:24,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:24,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:24,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:24,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:24,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788764003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:24,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:24,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:24,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:24,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:24,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788764003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:24,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:24,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:24,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:24,174 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=140, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/07d5ce375e6e4782ac819d9bc49af226 2024-12-09T23:58:24,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/2513aa2dc59f41c5a3078738d569457c is 50, key is test_row_0/B:col10/1733788702720/Put/seqid=0 2024-12-09T23:58:24,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742162_1338 (size=12151) 2024-12-09T23:58:24,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:24,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788764513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:24,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:24,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:24,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:24,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:24,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788764522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:24,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:24,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:24,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788764522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:24,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:24,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:24,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:24,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:24,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:24,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:24,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788764522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:24,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:24,527 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:24,527 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:24,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788764523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:24,527 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:24,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:24,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:24,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:24,585 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/2513aa2dc59f41c5a3078738d569457c 2024-12-09T23:58:24,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/321f3a5fb4f341db98854322e9e0d482 is 50, key is test_row_0/C:col10/1733788702720/Put/seqid=0 2024-12-09T23:58:24,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742163_1339 (size=12151) 2024-12-09T23:58:24,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T23:58:24,793 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:58:24,793 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:58:24,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-09T23:58:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T23:58:24,795 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:58:24,796 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:58:24,796 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:58:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T23:58:24,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-09T23:58:24,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:24,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:24,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:24,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:24,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:24,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:25,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/321f3a5fb4f341db98854322e9e0d482 2024-12-09T23:58:25,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/07d5ce375e6e4782ac819d9bc49af226 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/07d5ce375e6e4782ac819d9bc49af226 2024-12-09T23:58:25,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/07d5ce375e6e4782ac819d9bc49af226, entries=200, sequenceid=140, filesize=38.8 K 2024-12-09T23:58:25,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/2513aa2dc59f41c5a3078738d569457c as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/2513aa2dc59f41c5a3078738d569457c 2024-12-09T23:58:25,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/2513aa2dc59f41c5a3078738d569457c, entries=150, sequenceid=140, filesize=11.9 K 2024-12-09T23:58:25,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/321f3a5fb4f341db98854322e9e0d482 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/321f3a5fb4f341db98854322e9e0d482 2024-12-09T23:58:25,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/321f3a5fb4f341db98854322e9e0d482, entries=150, sequenceid=140, filesize=11.9 K 2024-12-09T23:58:25,014 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a41ba930d231212908b11d93742f8c4a in 1660ms, sequenceid=140, compaction requested=true 2024-12-09T23:58:25,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a41ba930d231212908b11d93742f8c4a: 2024-12-09T23:58:25,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a41ba930d231212908b11d93742f8c4a:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:58:25,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:25,014 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:58:25,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a41ba930d231212908b11d93742f8c4a:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:58:25,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:25,014 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:58:25,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a41ba930d231212908b11d93742f8c4a:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:58:25,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:58:25,015 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101899 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:58:25,015 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:58:25,015 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): a41ba930d231212908b11d93742f8c4a/A is initiating minor compaction (all files) 2024-12-09T23:58:25,015 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): a41ba930d231212908b11d93742f8c4a/B is initiating minor compaction (all files) 2024-12-09T23:58:25,015 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a41ba930d231212908b11d93742f8c4a/B in TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:25,015 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a41ba930d231212908b11d93742f8c4a/A in TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:25,015 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/9b489fa75fd54d0da70367020c43f08c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/0cf9439c07204f7b9c8fae1a3d8d00bd, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/07d5ce375e6e4782ac819d9bc49af226] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp, totalSize=99.5 K 2024-12-09T23:58:25,015 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/9e54be9e153f4fb2ad95f0bc6eb15fca, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/4c554236fbb04c48a2b2a0de3346915c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/2513aa2dc59f41c5a3078738d569457c] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp, totalSize=35.5 K 2024-12-09T23:58:25,015 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:25,015 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. files: [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/9b489fa75fd54d0da70367020c43f08c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/0cf9439c07204f7b9c8fae1a3d8d00bd, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/07d5ce375e6e4782ac819d9bc49af226] 2024-12-09T23:58:25,016 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e54be9e153f4fb2ad95f0bc6eb15fca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733788699876 2024-12-09T23:58:25,016 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b489fa75fd54d0da70367020c43f08c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733788699876 2024-12-09T23:58:25,016 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c554236fbb04c48a2b2a0de3346915c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733788700532 2024-12-09T23:58:25,016 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 2513aa2dc59f41c5a3078738d569457c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733788702719 2024-12-09T23:58:25,016 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cf9439c07204f7b9c8fae1a3d8d00bd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733788700532 2024-12-09T23:58:25,017 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07d5ce375e6e4782ac819d9bc49af226, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733788702719 2024-12-09T23:58:25,023 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:25,024 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a41ba930d231212908b11d93742f8c4a#B#compaction#288 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:25,025 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/4311dafb6a974a839c116696680221ea is 50, key is test_row_0/B:col10/1733788702720/Put/seqid=0 2024-12-09T23:58:25,026 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120996f25278d774420685943fcbeb200503_a41ba930d231212908b11d93742f8c4a store=[table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:25,028 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120996f25278d774420685943fcbeb200503_a41ba930d231212908b11d93742f8c4a, store=[table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:25,028 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120996f25278d774420685943fcbeb200503_a41ba930d231212908b11d93742f8c4a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:25,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742165_1341 (size=4469) 2024-12-09T23:58:25,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742164_1340 (size=12493) 2024-12-09T23:58:25,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-09T23:58:25,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:25,100 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing a41ba930d231212908b11d93742f8c4a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T23:58:25,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=A 2024-12-09T23:58:25,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:25,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=B 2024-12-09T23:58:25,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:25,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=C 2024-12-09T23:58:25,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:25,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099d3018ea13774997a8b0bb171b118631_a41ba930d231212908b11d93742f8c4a is 50, key is test_row_0/A:col10/1733788703373/Put/seqid=0 2024-12-09T23:58:25,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742166_1342 (size=12304) 2024-12-09T23:58:25,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T23:58:25,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T23:58:25,434 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a41ba930d231212908b11d93742f8c4a#A#compaction#289 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:25,435 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/f4cae3fd02b6439e8ac312f76033a823 is 175, key is test_row_0/A:col10/1733788702720/Put/seqid=0 2024-12-09T23:58:25,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742167_1343 (size=31447) 2024-12-09T23:58:25,438 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/4311dafb6a974a839c116696680221ea as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/4311dafb6a974a839c116696680221ea 2024-12-09T23:58:25,444 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a41ba930d231212908b11d93742f8c4a/B of a41ba930d231212908b11d93742f8c4a into 4311dafb6a974a839c116696680221ea(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:25,444 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a41ba930d231212908b11d93742f8c4a: 2024-12-09T23:58:25,444 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., storeName=a41ba930d231212908b11d93742f8c4a/B, priority=13, startTime=1733788705014; duration=0sec 2024-12-09T23:58:25,444 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/f4cae3fd02b6439e8ac312f76033a823 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/f4cae3fd02b6439e8ac312f76033a823 2024-12-09T23:58:25,444 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:58:25,444 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a41ba930d231212908b11d93742f8c4a:B 2024-12-09T23:58:25,444 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:58:25,445 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:58:25,445 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): a41ba930d231212908b11d93742f8c4a/C is initiating minor compaction (all files) 2024-12-09T23:58:25,445 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a41ba930d231212908b11d93742f8c4a/C in TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:25,446 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/a2955c9d85ce448dbe0701d6de7c4994, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/02c6e5e5657f43a9a4c2aaeeca0c688b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/321f3a5fb4f341db98854322e9e0d482] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp, totalSize=35.5 K 2024-12-09T23:58:25,446 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting a2955c9d85ce448dbe0701d6de7c4994, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733788699876 2024-12-09T23:58:25,446 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 02c6e5e5657f43a9a4c2aaeeca0c688b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733788700532 2024-12-09T23:58:25,447 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 321f3a5fb4f341db98854322e9e0d482, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733788702719 2024-12-09T23:58:25,449 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a41ba930d231212908b11d93742f8c4a/A of a41ba930d231212908b11d93742f8c4a into f4cae3fd02b6439e8ac312f76033a823(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:25,449 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a41ba930d231212908b11d93742f8c4a: 2024-12-09T23:58:25,449 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., storeName=a41ba930d231212908b11d93742f8c4a/A, priority=13, startTime=1733788705014; duration=0sec 2024-12-09T23:58:25,449 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:25,449 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a41ba930d231212908b11d93742f8c4a:A 2024-12-09T23:58:25,455 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a41ba930d231212908b11d93742f8c4a#C#compaction#291 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:25,456 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/0037fbbb61ed4a2287ecb1189cbe4c14 is 50, key is test_row_0/C:col10/1733788702720/Put/seqid=0 2024-12-09T23:58:25,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742168_1344 (size=12493) 2024-12-09T23:58:25,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:58:25,512 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099d3018ea13774997a8b0bb171b118631_a41ba930d231212908b11d93742f8c4a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099d3018ea13774997a8b0bb171b118631_a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:25,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/c8f581ca85f04637a4cee0af1fe0f15b, store: [table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:25,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/c8f581ca85f04637a4cee0af1fe0f15b is 175, key is test_row_0/A:col10/1733788703373/Put/seqid=0 2024-12-09T23:58:25,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742169_1345 (size=31105) 2024-12-09T23:58:25,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:25,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:25,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:25,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788765579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:25,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:25,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788765579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:25,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:25,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:25,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:25,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:25,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:25,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:25,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:25,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788765580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:25,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:25,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:25,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:25,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788765580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:25,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:25,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:25,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:25,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:25,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:25,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788765581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:25,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:25,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:25,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:25,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:25,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788765693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:25,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:25,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:25,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:25,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:25,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788765693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:25,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:25,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:25,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:25,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:25,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788765693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:25,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:25,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:25,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:25,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:25,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788765693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:25,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:25,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:25,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:25,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:25,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788765693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:25,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:25,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:25,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:25,865 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/0037fbbb61ed4a2287ecb1189cbe4c14 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/0037fbbb61ed4a2287ecb1189cbe4c14 2024-12-09T23:58:25,869 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a41ba930d231212908b11d93742f8c4a/C of a41ba930d231212908b11d93742f8c4a into 0037fbbb61ed4a2287ecb1189cbe4c14(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:25,869 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a41ba930d231212908b11d93742f8c4a: 2024-12-09T23:58:25,869 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., storeName=a41ba930d231212908b11d93742f8c4a/C, priority=13, startTime=1733788705014; duration=0sec 2024-12-09T23:58:25,869 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:25,869 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a41ba930d231212908b11d93742f8c4a:C 2024-12-09T23:58:25,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:25,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788765903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:25,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:25,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:25,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:25,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:25,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788765903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:25,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:25,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:25,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:25,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:25,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788765903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:25,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:25,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:25,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:25,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:25,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788765903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:25,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:25,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:25,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788765903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:25,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:25,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:25,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:25,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:25,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:25,917 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=152, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/c8f581ca85f04637a4cee0af1fe0f15b 2024-12-09T23:58:25,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/7b389894d9e642d497646c2a9ba20e8b is 50, key is test_row_0/B:col10/1733788703373/Put/seqid=0 2024-12-09T23:58:25,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742170_1346 (size=12151) 2024-12-09T23:58:25,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T23:58:26,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788766213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:26,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:26,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:26,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:26,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788766213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:26,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:26,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:26,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:26,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788766213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:26,218 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:26,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:26,218 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:26,218 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788766213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:26,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:26,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788766213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:26,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:26,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:26,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:26,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:26,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:26,327 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/7b389894d9e642d497646c2a9ba20e8b 2024-12-09T23:58:26,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/22d640ec7f6f443f82df260ffa71f73b is 50, key is test_row_0/C:col10/1733788703373/Put/seqid=0 2024-12-09T23:58:26,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742171_1347 (size=12151) 2024-12-09T23:58:26,338 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/22d640ec7f6f443f82df260ffa71f73b 2024-12-09T23:58:26,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/c8f581ca85f04637a4cee0af1fe0f15b as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/c8f581ca85f04637a4cee0af1fe0f15b 2024-12-09T23:58:26,345 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/c8f581ca85f04637a4cee0af1fe0f15b, entries=150, sequenceid=152, filesize=30.4 K 2024-12-09T23:58:26,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/7b389894d9e642d497646c2a9ba20e8b as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/7b389894d9e642d497646c2a9ba20e8b 2024-12-09T23:58:26,349 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/7b389894d9e642d497646c2a9ba20e8b, entries=150, sequenceid=152, filesize=11.9 K 2024-12-09T23:58:26,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/22d640ec7f6f443f82df260ffa71f73b as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/22d640ec7f6f443f82df260ffa71f73b 2024-12-09T23:58:26,354 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/22d640ec7f6f443f82df260ffa71f73b, entries=150, sequenceid=152, filesize=11.9 K 2024-12-09T23:58:26,396 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a41ba930d231212908b11d93742f8c4a in 1295ms, sequenceid=152, compaction requested=false 2024-12-09T23:58:26,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for a41ba930d231212908b11d93742f8c4a: 2024-12-09T23:58:26,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:26,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-09T23:58:26,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-09T23:58:26,398 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-09T23:58:26,398 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6010 sec 2024-12-09T23:58:26,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.6060 sec 2024-12-09T23:58:26,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:26,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a41ba930d231212908b11d93742f8c4a 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-09T23:58:26,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=A 2024-12-09T23:58:26,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733788705578, max timestamp=1733788706725Num uniques 859; 2024-12-09T23:58:26,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733788705578, max timestamp=1733788706725Num uniques 859; 2024-12-09T23:58:26,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733788705578, max timestamp=1733788706725Num uniques 859; 2024-12-09T23:58:26,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T23:58:26,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T23:58:26,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T23:58:26,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:58:26,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=B 2024-12-09T23:58:26,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:58:26,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=C 2024-12-09T23:58:26,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:58:26,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209e0ff9dd8df654f92a3fbb03ea5d6c98b_a41ba930d231212908b11d93742f8c4a is 50, key is test_row_0/A:col10/1733788706726/Put/seqid=0 2024-12-09T23:58:26,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742172_1348 (size=14794) 2024-12-09T23:58:26,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733788706748, max timestamp=1733788706761Num uniques 859; 2024-12-09T23:58:26,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T23:58:26,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733788706748, max timestamp=1733788706761Num uniques 859; 2024-12-09T23:58:26,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T23:58:26,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733788706748, max timestamp=1733788706761Num uniques 859; 2024-12-09T23:58:26,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T23:58:26,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:26,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788766788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:26,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:26,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788766790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:26,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788766801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:26,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733788706772, max timestamp=1733788706800Num uniques 1118; 2024-12-09T23:58:26,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-12-09T23:58:26,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733788706772, max timestamp=1733788706800Num uniques 1118; 2024-12-09T23:58:26,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:26,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-12-09T23:58:26,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:26,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:26,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733788706772, max timestamp=1733788706800Num uniques 1118; 2024-12-09T23:58:26,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-12-09T23:58:26,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:26,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:26,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:26,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:26,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:26,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:26,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733788706856, max timestamp=1733788706888Num uniques 1127; 2024-12-09T23:58:26,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-12-09T23:58:26,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733788706856, max timestamp=1733788706888Num uniques 1127; 2024-12-09T23:58:26,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-12-09T23:58:26,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733788706856, max timestamp=1733788706888Num uniques 1127; 2024-12-09T23:58:26,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-12-09T23:58:26,933 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=386, cellSize=17676, totalHeapSize=40384, min timestamp=1733788706873, max timestamp=1733788706930Num uniques 1086; 2024-12-09T23:58:26,933 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=17676, suffix heap size=140912,new segment heap size=40384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=386 2024-12-09T23:58:26,933 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=386, cellSize=17676, totalHeapSize=40384, min timestamp=1733788706873, max timestamp=1733788706930Num uniques 1086; 2024-12-09T23:58:26,933 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=17676, suffix heap size=140912,new segment heap size=40384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=386 2024-12-09T23:58:26,933 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=386, cellSize=17676, totalHeapSize=40384, min timestamp=1733788706873, max timestamp=1733788706930Num uniques 1086; 2024-12-09T23:58:26,933 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=17676, suffix heap size=140912,new segment heap size=40384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=386 2024-12-09T23:58:26,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T23:58:26,943 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:58:26,944 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:58:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-09T23:58:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T23:58:26,945 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:58:26,945 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:58:26,946 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:58:26,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788766961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:26,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:26,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:26,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788766962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:26,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:26,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:26,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:26,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:26,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788766963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:26,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:26,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:26,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:26,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788766964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:26,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:26,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:26,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788766966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T23:58:27,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788767082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:27,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:27,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:27,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:27,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788767082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:27,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:27,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:27,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:27,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788767082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:27,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:27,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:27,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:27,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788767082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:27,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:27,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788767083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:27,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:27,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:27,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:27,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:27,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:27,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-09T23:58:27,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:27,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:27,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:27,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:27,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:27,163 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:58:27,166 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209e0ff9dd8df654f92a3fbb03ea5d6c98b_a41ba930d231212908b11d93742f8c4a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e0ff9dd8df654f92a3fbb03ea5d6c98b_a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:27,167 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/2f45f45dcc5c4f78b2c38320b81ec3cb, store: [table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:27,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/2f45f45dcc5c4f78b2c38320b81ec3cb is 175, key is test_row_0/A:col10/1733788706726/Put/seqid=0 2024-12-09T23:58:27,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742173_1349 (size=39749) 2024-12-09T23:58:27,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-09T23:58:27,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:27,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:27,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:27,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:27,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:27,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T23:58:27,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:27,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788767292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:27,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:27,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:27,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:27,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:27,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788767293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:27,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:27,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:27,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:27,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:27,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788767293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:27,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:27,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788767293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:27,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:27,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:27,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:27,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788767293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:27,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:27,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:27,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:27,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:27,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:27,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-09T23:58:27,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:27,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:27,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:27,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:27,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:27,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:27,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-09T23:58:27,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:27,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:27,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:27,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:27,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:27,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:27,572 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=177, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/2f45f45dcc5c4f78b2c38320b81ec3cb 2024-12-09T23:58:27,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T23:58:27,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/04e99d97650c4254be90667eb31f4ec2 is 50, key is test_row_0/B:col10/1733788706726/Put/seqid=0 2024-12-09T23:58:27,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742174_1350 (size=12151) 2024-12-09T23:58:27,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788767602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:27,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:27,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:27,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:27,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:27,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788767613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:27,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:27,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:27,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788767613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:27,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:27,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:27,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:27,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:27,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:27,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:27,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788767613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:27,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:27,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:27,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:27,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:27,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788767613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:27,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:27,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:27,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:27,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-09T23:58:27,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:27,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:27,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:27,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:27,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:27,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:27,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-09T23:58:27,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:27,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:27,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:27,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:27,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:27,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/04e99d97650c4254be90667eb31f4ec2 2024-12-09T23:58:27,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/cb779f9c7dc340c5bdf9131e11468d60 is 50, key is test_row_0/C:col10/1733788706726/Put/seqid=0 2024-12-09T23:58:27,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742175_1351 (size=12151) 2024-12-09T23:58:28,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-09T23:58:28,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:28,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:28,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:28,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:28,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:28,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:28,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T23:58:28,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:28,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788768123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:28,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:28,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:28,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:28,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788768133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:28,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:28,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788768133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:28,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:28,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788768133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:28,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:28,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788768133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:28,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-09T23:58:28,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:28,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:28,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:28,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:28,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:28,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:28,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-09T23:58:28,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:28,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:28,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:28,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:28,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:28,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:28,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/cb779f9c7dc340c5bdf9131e11468d60 2024-12-09T23:58:28,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/2f45f45dcc5c4f78b2c38320b81ec3cb as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/2f45f45dcc5c4f78b2c38320b81ec3cb 2024-12-09T23:58:28,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/2f45f45dcc5c4f78b2c38320b81ec3cb, entries=200, sequenceid=177, filesize=38.8 K 2024-12-09T23:58:28,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/04e99d97650c4254be90667eb31f4ec2 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/04e99d97650c4254be90667eb31f4ec2 2024-12-09T23:58:28,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/04e99d97650c4254be90667eb31f4ec2, entries=150, sequenceid=177, filesize=11.9 K 2024-12-09T23:58:28,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/cb779f9c7dc340c5bdf9131e11468d60 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/cb779f9c7dc340c5bdf9131e11468d60 2024-12-09T23:58:28,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/cb779f9c7dc340c5bdf9131e11468d60, entries=150, sequenceid=177, filesize=11.9 K 2024-12-09T23:58:28,413 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=161.02 KB/164880 for a41ba930d231212908b11d93742f8c4a in 1687ms, sequenceid=177, compaction requested=true 2024-12-09T23:58:28,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a41ba930d231212908b11d93742f8c4a: 2024-12-09T23:58:28,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a41ba930d231212908b11d93742f8c4a:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:58:28,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:28,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a41ba930d231212908b11d93742f8c4a:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:58:28,413 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:58:28,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:28,413 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:58:28,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a41ba930d231212908b11d93742f8c4a:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:58:28,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:58:28,414 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:58:28,414 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:58:28,414 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): a41ba930d231212908b11d93742f8c4a/B is initiating minor compaction (all files) 2024-12-09T23:58:28,414 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): a41ba930d231212908b11d93742f8c4a/A is initiating minor compaction (all files) 2024-12-09T23:58:28,414 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a41ba930d231212908b11d93742f8c4a/B in TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:28,414 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a41ba930d231212908b11d93742f8c4a/A in TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:28,414 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/4311dafb6a974a839c116696680221ea, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/7b389894d9e642d497646c2a9ba20e8b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/04e99d97650c4254be90667eb31f4ec2] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp, totalSize=35.9 K 2024-12-09T23:58:28,414 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/f4cae3fd02b6439e8ac312f76033a823, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/c8f581ca85f04637a4cee0af1fe0f15b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/2f45f45dcc5c4f78b2c38320b81ec3cb] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp, totalSize=99.9 K 2024-12-09T23:58:28,414 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:28,414 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. files: [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/f4cae3fd02b6439e8ac312f76033a823, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/c8f581ca85f04637a4cee0af1fe0f15b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/2f45f45dcc5c4f78b2c38320b81ec3cb] 2024-12-09T23:58:28,414 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 4311dafb6a974a839c116696680221ea, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733788702719 2024-12-09T23:58:28,414 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4cae3fd02b6439e8ac312f76033a823, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733788702719 2024-12-09T23:58:28,415 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b389894d9e642d497646c2a9ba20e8b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733788703365 2024-12-09T23:58:28,415 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8f581ca85f04637a4cee0af1fe0f15b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733788703365 2024-12-09T23:58:28,415 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 04e99d97650c4254be90667eb31f4ec2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733788705579 2024-12-09T23:58:28,415 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f45f45dcc5c4f78b2c38320b81ec3cb, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733788705578 2024-12-09T23:58:28,420 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:28,421 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a41ba930d231212908b11d93742f8c4a#B#compaction#297 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:28,422 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/63bd34ea453d4202890328433fa9a8ad is 50, key is test_row_0/B:col10/1733788706726/Put/seqid=0 2024-12-09T23:58:28,424 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209463f315b8b2e499295455af8432653e7_a41ba930d231212908b11d93742f8c4a store=[table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:28,426 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209463f315b8b2e499295455af8432653e7_a41ba930d231212908b11d93742f8c4a, store=[table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:28,426 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209463f315b8b2e499295455af8432653e7_a41ba930d231212908b11d93742f8c4a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:28,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742176_1352 (size=12595) 2024-12-09T23:58:28,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742177_1353 (size=4469) 2024-12-09T23:58:28,443 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/63bd34ea453d4202890328433fa9a8ad as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/63bd34ea453d4202890328433fa9a8ad 2024-12-09T23:58:28,447 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a41ba930d231212908b11d93742f8c4a/B of a41ba930d231212908b11d93742f8c4a into 63bd34ea453d4202890328433fa9a8ad(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:28,447 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a41ba930d231212908b11d93742f8c4a: 2024-12-09T23:58:28,447 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., storeName=a41ba930d231212908b11d93742f8c4a/B, priority=13, startTime=1733788708413; duration=0sec 2024-12-09T23:58:28,447 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:58:28,447 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a41ba930d231212908b11d93742f8c4a:B 2024-12-09T23:58:28,447 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:58:28,448 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:58:28,448 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): a41ba930d231212908b11d93742f8c4a/C is initiating minor compaction (all files) 2024-12-09T23:58:28,448 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a41ba930d231212908b11d93742f8c4a/C in TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:28,448 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/0037fbbb61ed4a2287ecb1189cbe4c14, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/22d640ec7f6f443f82df260ffa71f73b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/cb779f9c7dc340c5bdf9131e11468d60] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp, totalSize=35.9 K 2024-12-09T23:58:28,448 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 0037fbbb61ed4a2287ecb1189cbe4c14, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733788702719 2024-12-09T23:58:28,449 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 22d640ec7f6f443f82df260ffa71f73b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733788703365 2024-12-09T23:58:28,449 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting cb779f9c7dc340c5bdf9131e11468d60, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733788705579 2024-12-09T23:58:28,453 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a41ba930d231212908b11d93742f8c4a#C#compaction#299 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:28,454 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/b998699045894360955c2a4337f830b6 is 50, key is test_row_0/C:col10/1733788706726/Put/seqid=0 2024-12-09T23:58:28,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742178_1354 (size=12595) 2024-12-09T23:58:28,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-09T23:58:28,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:28,471 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing a41ba930d231212908b11d93742f8c4a 3/3 column families, dataSize=161.02 KB heapSize=443.39 KB 2024-12-09T23:58:28,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=A 2024-12-09T23:58:28,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:58:28,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=B 2024-12-09T23:58:28,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:58:28,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=C 2024-12-09T23:58:28,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:58:28,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120986ac137acce449a38560fe816f326130_a41ba930d231212908b11d93742f8c4a is 50, key is test_row_0/A:col10/1733788706960/Put/seqid=0 2024-12-09T23:58:28,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742179_1355 (size=12454) 2024-12-09T23:58:28,841 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a41ba930d231212908b11d93742f8c4a#A#compaction#298 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:28,842 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/783827b3fb9b4f63bd297a660d957c27 is 175, key is test_row_0/A:col10/1733788706726/Put/seqid=0 2024-12-09T23:58:28,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742180_1356 (size=31549) 2024-12-09T23:58:28,861 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/b998699045894360955c2a4337f830b6 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/b998699045894360955c2a4337f830b6 2024-12-09T23:58:28,865 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a41ba930d231212908b11d93742f8c4a/C of a41ba930d231212908b11d93742f8c4a into b998699045894360955c2a4337f830b6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:28,865 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a41ba930d231212908b11d93742f8c4a: 2024-12-09T23:58:28,865 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., storeName=a41ba930d231212908b11d93742f8c4a/C, priority=13, startTime=1733788708413; duration=0sec 2024-12-09T23:58:28,865 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:28,865 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a41ba930d231212908b11d93742f8c4a:C 2024-12-09T23:58:28,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:58:28,885 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120986ac137acce449a38560fe816f326130_a41ba930d231212908b11d93742f8c4a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120986ac137acce449a38560fe816f326130_a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:28,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/83379ef2c2db4dd5a3ef09505b639c78, store: [table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:28,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/83379ef2c2db4dd5a3ef09505b639c78 is 175, key is test_row_0/A:col10/1733788706960/Put/seqid=0 2024-12-09T23:58:28,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742181_1357 (size=31255) 2024-12-09T23:58:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T23:58:29,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:29,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:29,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788769153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:29,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788769153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:29,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788769153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:29,159 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_1' in a41ba930d231212908b11d93742f8c4a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:58:29,160 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_0' in a41ba930d231212908b11d93742f8c4a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:58:29,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:29,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:29,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:29,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:29,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:29,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:29,160 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_1' in a41ba930d231212908b11d93742f8c4a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:58:29,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:29,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:29,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:29,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788769157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:29,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:29,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:29,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:29,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788769159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:29,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:29,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:29,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:29,254 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/783827b3fb9b4f63bd297a660d957c27 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/783827b3fb9b4f63bd297a660d957c27 2024-12-09T23:58:29,257 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a41ba930d231212908b11d93742f8c4a/A of a41ba930d231212908b11d93742f8c4a into 783827b3fb9b4f63bd297a660d957c27(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:29,257 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a41ba930d231212908b11d93742f8c4a: 2024-12-09T23:58:29,257 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., storeName=a41ba930d231212908b11d93742f8c4a/A, priority=13, startTime=1733788708413; duration=0sec 2024-12-09T23:58:29,257 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:29,257 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a41ba930d231212908b11d93742f8c4a:A 2024-12-09T23:58:29,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788769273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:29,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788769273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:29,290 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=266, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/83379ef2c2db4dd5a3ef09505b639c78 2024-12-09T23:58:29,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/074285c26b2f4232bb9e5a1744fc1dcc is 50, key is test_row_0/B:col10/1733788706960/Put/seqid=0 2024-12-09T23:58:29,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742182_1358 (size=12301) 2024-12-09T23:58:29,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788769483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:29,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:29,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:29,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:29,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788769483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:29,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:29,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:29,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:29,701 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/074285c26b2f4232bb9e5a1744fc1dcc 2024-12-09T23:58:29,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/4de0bf0dfcec44a89a9b64a351ae3f6a is 50, key is test_row_0/C:col10/1733788706960/Put/seqid=0 2024-12-09T23:58:29,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742183_1359 (size=12301) 2024-12-09T23:58:29,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:29,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788769792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:29,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:29,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788769792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:30,112 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/4de0bf0dfcec44a89a9b64a351ae3f6a 2024-12-09T23:58:30,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/83379ef2c2db4dd5a3ef09505b639c78 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/83379ef2c2db4dd5a3ef09505b639c78 2024-12-09T23:58:30,120 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/83379ef2c2db4dd5a3ef09505b639c78, entries=150, sequenceid=266, filesize=30.5 K 2024-12-09T23:58:30,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/074285c26b2f4232bb9e5a1744fc1dcc as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/074285c26b2f4232bb9e5a1744fc1dcc 2024-12-09T23:58:30,124 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/074285c26b2f4232bb9e5a1744fc1dcc, entries=150, sequenceid=266, filesize=12.0 K 2024-12-09T23:58:30,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/4de0bf0dfcec44a89a9b64a351ae3f6a as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/4de0bf0dfcec44a89a9b64a351ae3f6a 2024-12-09T23:58:30,129 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/4de0bf0dfcec44a89a9b64a351ae3f6a, entries=150, sequenceid=266, filesize=12.0 K 2024-12-09T23:58:30,130 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~443.34 KB/453984, currentSize=26.84 KB/27480 for a41ba930d231212908b11d93742f8c4a in 1659ms, sequenceid=266, compaction requested=false 2024-12-09T23:58:30,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for a41ba930d231212908b11d93742f8c4a: 2024-12-09T23:58:30,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:30,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-09T23:58:30,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-09T23:58:30,132 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-09T23:58:30,132 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1850 sec 2024-12-09T23:58:30,134 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 3.1890 sec 2024-12-09T23:58:30,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:30,313 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a41ba930d231212908b11d93742f8c4a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T23:58:30,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=A 2024-12-09T23:58:30,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:30,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=B 2024-12-09T23:58:30,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:30,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=C 2024-12-09T23:58:30,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:30,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209124370a7224946a281a3df7bce8cc830_a41ba930d231212908b11d93742f8c4a is 50, key is test_row_0/A:col10/1733788710311/Put/seqid=0 2024-12-09T23:58:30,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742184_1360 (size=14994) 2024-12-09T23:58:30,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733788710343, max timestamp=1733788710359Num uniques 859; 2024-12-09T23:58:30,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733788710343, max timestamp=1733788710359Num uniques 859; 2024-12-09T23:58:30,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T23:58:30,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T23:58:30,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733788710343, max timestamp=1733788710359Num uniques 859; 2024-12-09T23:58:30,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T23:58:30,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:30,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788770414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:30,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:30,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:30,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788770415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:30,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:30,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:30,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:30,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:30,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:30,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:30,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788770533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:30,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:30,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788770533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:30,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:30,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:30,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:30,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:30,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:30,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:30,721 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:58:30,725 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209124370a7224946a281a3df7bce8cc830_a41ba930d231212908b11d93742f8c4a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209124370a7224946a281a3df7bce8cc830_a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:30,725 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/aed34e7e81224f079ce0fd98293cb91e, store: [table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:30,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/aed34e7e81224f079ce0fd98293cb91e is 175, key is test_row_0/A:col10/1733788710311/Put/seqid=0 2024-12-09T23:58:30,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742185_1361 (size=39949) 2024-12-09T23:58:30,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:30,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788770742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:30,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:30,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788770743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:30,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:30,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:30,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:30,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:30,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:30,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:31,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788771052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:31,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:31,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:31,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:31,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788771052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:31,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:31,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:31,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:31,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T23:58:31,113 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:58:31,114 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:58:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-09T23:58:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-09T23:58:31,115 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:58:31,115 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:58:31,115 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:58:31,129 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=278, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/aed34e7e81224f079ce0fd98293cb91e 2024-12-09T23:58:31,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/26996ff0b1b74aac9bf8d06916b9880b is 50, key is test_row_0/B:col10/1733788710311/Put/seqid=0 2024-12-09T23:58:31,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742186_1362 (size=12301) 2024-12-09T23:58:31,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:31,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788771172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:31,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:31,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788771172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:31,178 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_0' in a41ba930d231212908b11d93742f8c4a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:58:31,178 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_1' in a41ba930d231212908b11d93742f8c4a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:58:31,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:31,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:31,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:31,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:31,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:31,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:31,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:31,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788771183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:31,187 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_1' in a41ba930d231212908b11d93742f8c4a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:58:31,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:31,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:31,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:31,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-09T23:58:31,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-09T23:58:31,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:31,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:31,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:31,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:31,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:31,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:31,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-09T23:58:31,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:31,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:31,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:31,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:31,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-09T23:58:31,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/26996ff0b1b74aac9bf8d06916b9880b 2024-12-09T23:58:31,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/353db2e1ba054776adc3b178b78cf683 is 50, key is test_row_0/C:col10/1733788710311/Put/seqid=0 2024-12-09T23:58:31,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742187_1363 (size=12301) 2024-12-09T23:58:31,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-09T23:58:31,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:31,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:31,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:31,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:31,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:31,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:31,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788771573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:31,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:31,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:31,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788771573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:31,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:31,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:31,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:31,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:31,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-09T23:58:31,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:31,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:31,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:31,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:31,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:31,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:31,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-09T23:58:31,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-09T23:58:31,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:31,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:31,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:31,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:31,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:31,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:31,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/353db2e1ba054776adc3b178b78cf683 2024-12-09T23:58:31,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/aed34e7e81224f079ce0fd98293cb91e as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/aed34e7e81224f079ce0fd98293cb91e 2024-12-09T23:58:31,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/aed34e7e81224f079ce0fd98293cb91e, entries=200, sequenceid=278, filesize=39.0 K 2024-12-09T23:58:31,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/26996ff0b1b74aac9bf8d06916b9880b as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/26996ff0b1b74aac9bf8d06916b9880b 2024-12-09T23:58:31,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/26996ff0b1b74aac9bf8d06916b9880b, entries=150, sequenceid=278, filesize=12.0 K 2024-12-09T23:58:31,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/353db2e1ba054776adc3b178b78cf683 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/353db2e1ba054776adc3b178b78cf683 2024-12-09T23:58:31,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/353db2e1ba054776adc3b178b78cf683, entries=150, sequenceid=278, filesize=12.0 K 2024-12-09T23:58:31,965 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for a41ba930d231212908b11d93742f8c4a in 1653ms, sequenceid=278, compaction requested=true 2024-12-09T23:58:31,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a41ba930d231212908b11d93742f8c4a: 2024-12-09T23:58:31,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a41ba930d231212908b11d93742f8c4a:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:58:31,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:31,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a41ba930d231212908b11d93742f8c4a:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:58:31,965 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:58:31,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:31,965 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:58:31,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a41ba930d231212908b11d93742f8c4a:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:58:31,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:58:31,966 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:58:31,966 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102753 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:58:31,966 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): a41ba930d231212908b11d93742f8c4a/B is initiating minor compaction (all files) 2024-12-09T23:58:31,966 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): a41ba930d231212908b11d93742f8c4a/A is initiating minor compaction (all files) 2024-12-09T23:58:31,966 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a41ba930d231212908b11d93742f8c4a/B in TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:31,966 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a41ba930d231212908b11d93742f8c4a/A in TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:31,966 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/63bd34ea453d4202890328433fa9a8ad, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/074285c26b2f4232bb9e5a1744fc1dcc, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/26996ff0b1b74aac9bf8d06916b9880b] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp, totalSize=36.3 K 2024-12-09T23:58:31,966 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/783827b3fb9b4f63bd297a660d957c27, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/83379ef2c2db4dd5a3ef09505b639c78, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/aed34e7e81224f079ce0fd98293cb91e] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp, totalSize=100.3 K 2024-12-09T23:58:31,966 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:31,966 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. files: [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/783827b3fb9b4f63bd297a660d957c27, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/83379ef2c2db4dd5a3ef09505b639c78, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/aed34e7e81224f079ce0fd98293cb91e] 2024-12-09T23:58:31,966 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 63bd34ea453d4202890328433fa9a8ad, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733788705579 2024-12-09T23:58:31,966 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 783827b3fb9b4f63bd297a660d957c27, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733788705579 2024-12-09T23:58:31,966 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 074285c26b2f4232bb9e5a1744fc1dcc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733788706960 2024-12-09T23:58:31,967 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83379ef2c2db4dd5a3ef09505b639c78, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733788706960 2024-12-09T23:58:31,967 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 26996ff0b1b74aac9bf8d06916b9880b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733788709151 2024-12-09T23:58:31,967 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting aed34e7e81224f079ce0fd98293cb91e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733788709151 2024-12-09T23:58:31,971 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:31,972 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a41ba930d231212908b11d93742f8c4a#B#compaction#306 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:31,973 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/cd535524bb66440cb8ba280ed327bfad is 50, key is test_row_0/B:col10/1733788710311/Put/seqid=0 2024-12-09T23:58:31,975 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412099f346a78b53f490a8ca6dc408068476b_a41ba930d231212908b11d93742f8c4a store=[table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:31,977 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412099f346a78b53f490a8ca6dc408068476b_a41ba930d231212908b11d93742f8c4a, store=[table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:31,977 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099f346a78b53f490a8ca6dc408068476b_a41ba930d231212908b11d93742f8c4a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:31,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742188_1364 (size=12847) 2024-12-09T23:58:31,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742189_1365 (size=4469) 2024-12-09T23:58:31,993 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a41ba930d231212908b11d93742f8c4a#A#compaction#307 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:31,994 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/5708e58540fa4c75af957e736abfb91b is 175, key is test_row_0/A:col10/1733788710311/Put/seqid=0 2024-12-09T23:58:32,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742190_1366 (size=31801) 2024-12-09T23:58:32,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-09T23:58:32,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:32,030 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing a41ba930d231212908b11d93742f8c4a 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-09T23:58:32,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=A 2024-12-09T23:58:32,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:58:32,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=B 2024-12-09T23:58:32,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:58:32,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=C 2024-12-09T23:58:32,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:58:32,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092afb060746f849caab6229f5f8ccfd96_a41ba930d231212908b11d93742f8c4a is 50, key is test_row_0/A:col10/1733788710409/Put/seqid=0 2024-12-09T23:58:32,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742191_1367 (size=12454) 2024-12-09T23:58:32,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:58:32,044 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092afb060746f849caab6229f5f8ccfd96_a41ba930d231212908b11d93742f8c4a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092afb060746f849caab6229f5f8ccfd96_a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:32,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/1013ac21ff354e79982dd8a3166e766c, store: [table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:32,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/1013ac21ff354e79982dd8a3166e766c is 175, key is test_row_0/A:col10/1733788710409/Put/seqid=0 2024-12-09T23:58:32,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742192_1368 (size=31255) 2024-12-09T23:58:32,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-09T23:58:32,384 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/cd535524bb66440cb8ba280ed327bfad as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/cd535524bb66440cb8ba280ed327bfad 2024-12-09T23:58:32,388 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a41ba930d231212908b11d93742f8c4a/B of a41ba930d231212908b11d93742f8c4a into cd535524bb66440cb8ba280ed327bfad(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:32,388 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a41ba930d231212908b11d93742f8c4a: 2024-12-09T23:58:32,388 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., storeName=a41ba930d231212908b11d93742f8c4a/B, priority=13, startTime=1733788711965; duration=0sec 2024-12-09T23:58:32,388 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:58:32,388 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a41ba930d231212908b11d93742f8c4a:B 2024-12-09T23:58:32,388 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:58:32,389 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:58:32,389 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): a41ba930d231212908b11d93742f8c4a/C is initiating minor compaction (all files) 2024-12-09T23:58:32,389 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a41ba930d231212908b11d93742f8c4a/C in TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:32,389 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/b998699045894360955c2a4337f830b6, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/4de0bf0dfcec44a89a9b64a351ae3f6a, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/353db2e1ba054776adc3b178b78cf683] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp, totalSize=36.3 K 2024-12-09T23:58:32,389 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting b998699045894360955c2a4337f830b6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733788705579 2024-12-09T23:58:32,389 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 4de0bf0dfcec44a89a9b64a351ae3f6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733788706960 2024-12-09T23:58:32,390 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 353db2e1ba054776adc3b178b78cf683, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733788709151 2024-12-09T23:58:32,395 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a41ba930d231212908b11d93742f8c4a#C#compaction#309 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:32,396 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/862b21811cdd4d0fa931d2a40339821c is 50, key is test_row_0/C:col10/1733788710311/Put/seqid=0 2024-12-09T23:58:32,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742193_1369 (size=12847) 2024-12-09T23:58:32,403 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/5708e58540fa4c75af957e736abfb91b as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/5708e58540fa4c75af957e736abfb91b 2024-12-09T23:58:32,407 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a41ba930d231212908b11d93742f8c4a/A of a41ba930d231212908b11d93742f8c4a into 5708e58540fa4c75af957e736abfb91b(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:32,407 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a41ba930d231212908b11d93742f8c4a: 2024-12-09T23:58:32,407 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., storeName=a41ba930d231212908b11d93742f8c4a/A, priority=13, startTime=1733788711965; duration=0sec 2024-12-09T23:58:32,407 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:32,407 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a41ba930d231212908b11d93742f8c4a:A 2024-12-09T23:58:32,449 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=314, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/1013ac21ff354e79982dd8a3166e766c 2024-12-09T23:58:32,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/b973bdcf8e1c4c5690999488da6ac585 is 50, key is test_row_0/B:col10/1733788710409/Put/seqid=0 2024-12-09T23:58:32,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742194_1370 (size=12301) 2024-12-09T23:58:32,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:32,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:32,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:32,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788772608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:32,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:32,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:32,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:32,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:32,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788772609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:32,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:32,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:32,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:32,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:32,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788772722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:32,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:32,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:32,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:32,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:32,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788772722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:32,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:32,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:32,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:32,803 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/862b21811cdd4d0fa931d2a40339821c as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/862b21811cdd4d0fa931d2a40339821c 2024-12-09T23:58:32,806 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a41ba930d231212908b11d93742f8c4a/C of a41ba930d231212908b11d93742f8c4a into 862b21811cdd4d0fa931d2a40339821c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:32,806 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a41ba930d231212908b11d93742f8c4a: 2024-12-09T23:58:32,807 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., storeName=a41ba930d231212908b11d93742f8c4a/C, priority=13, startTime=1733788711965; duration=0sec 2024-12-09T23:58:32,807 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:32,807 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a41ba930d231212908b11d93742f8c4a:C 2024-12-09T23:58:32,859 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/b973bdcf8e1c4c5690999488da6ac585 2024-12-09T23:58:32,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/84b238a61a8441798c9c45f313ea06d9 is 50, key is test_row_0/C:col10/1733788710409/Put/seqid=0 2024-12-09T23:58:32,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742195_1371 (size=12301) 2024-12-09T23:58:32,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:32,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788772933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:32,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:32,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:32,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:32,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:32,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788772933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:32,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:32,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:32,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:33,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:33,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788773242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:33,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:33,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:33,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788773242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:33,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:33,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:33,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:33,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:33,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:33,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-09T23:58:33,267 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/84b238a61a8441798c9c45f313ea06d9 2024-12-09T23:58:33,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/1013ac21ff354e79982dd8a3166e766c as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/1013ac21ff354e79982dd8a3166e766c 2024-12-09T23:58:33,274 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/1013ac21ff354e79982dd8a3166e766c, entries=150, sequenceid=314, filesize=30.5 K 2024-12-09T23:58:33,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/b973bdcf8e1c4c5690999488da6ac585 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/b973bdcf8e1c4c5690999488da6ac585 2024-12-09T23:58:33,278 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/b973bdcf8e1c4c5690999488da6ac585, entries=150, sequenceid=314, filesize=12.0 K 2024-12-09T23:58:33,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/84b238a61a8441798c9c45f313ea06d9 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/84b238a61a8441798c9c45f313ea06d9 2024-12-09T23:58:33,281 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/84b238a61a8441798c9c45f313ea06d9, entries=150, sequenceid=314, filesize=12.0 K 2024-12-09T23:58:33,282 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for a41ba930d231212908b11d93742f8c4a in 1252ms, sequenceid=314, compaction requested=false 2024-12-09T23:58:33,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for a41ba930d231212908b11d93742f8c4a: 2024-12-09T23:58:33,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:33,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-09T23:58:33,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-09T23:58:33,284 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-09T23:58:33,284 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1680 sec 2024-12-09T23:58:33,285 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.1700 sec 2024-12-09T23:58:33,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:33,756 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a41ba930d231212908b11d93742f8c4a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T23:58:33,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=A 2024-12-09T23:58:33,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:33,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=B 2024-12-09T23:58:33,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:33,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=C 2024-12-09T23:58:33,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:33,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ab70d53c84fe4d92b82ec410afc4e655_a41ba930d231212908b11d93742f8c4a is 50, key is test_row_0/A:col10/1733788713755/Put/seqid=0 2024-12-09T23:58:33,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742196_1372 (size=14994) 2024-12-09T23:58:33,773 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:58:33,777 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ab70d53c84fe4d92b82ec410afc4e655_a41ba930d231212908b11d93742f8c4a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ab70d53c84fe4d92b82ec410afc4e655_a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:33,778 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/4fecf98e0c48448c87dd67d97c845a46, store: [table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:33,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/4fecf98e0c48448c87dd67d97c845a46 is 175, key is test_row_0/A:col10/1733788713755/Put/seqid=0 2024-12-09T23:58:33,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742197_1373 (size=39949) 2024-12-09T23:58:33,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733788713786, max timestamp=1733788713812Num uniques 859; 2024-12-09T23:58:33,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733788713786, max timestamp=1733788713812Num uniques 859; 2024-12-09T23:58:33,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T23:58:33,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T23:58:33,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733788713786, max timestamp=1733788713812Num uniques 859; 2024-12-09T23:58:33,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T23:58:33,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:33,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788773887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:33,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:33,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788773891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:34,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:34,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788774003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:34,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:34,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:34,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:34,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788774003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:34,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:34,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:34,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:34,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:34,197 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=328, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/4fecf98e0c48448c87dd67d97c845a46 2024-12-09T23:58:34,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/822527172202456dbdddd667ea5c7b72 is 50, key is test_row_0/B:col10/1733788713755/Put/seqid=0 2024-12-09T23:58:34,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742198_1374 (size=12301) 2024-12-09T23:58:34,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:34,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788774215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:34,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:34,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:34,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:34,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:34,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788774223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:34,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:34,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:34,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:34,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:34,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788774533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:34,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:34,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:34,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:34,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:34,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788774533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:34,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:34,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:34,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:34,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/822527172202456dbdddd667ea5c7b72 2024-12-09T23:58:34,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/1c8f2c3852d54fc585c1602d18349924 is 50, key is test_row_0/C:col10/1733788713755/Put/seqid=0 2024-12-09T23:58:34,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742199_1375 (size=12301) 2024-12-09T23:58:35,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/1c8f2c3852d54fc585c1602d18349924 2024-12-09T23:58:35,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/4fecf98e0c48448c87dd67d97c845a46 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/4fecf98e0c48448c87dd67d97c845a46 2024-12-09T23:58:35,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/4fecf98e0c48448c87dd67d97c845a46, entries=200, sequenceid=328, filesize=39.0 K 2024-12-09T23:58:35,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/822527172202456dbdddd667ea5c7b72 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/822527172202456dbdddd667ea5c7b72 2024-12-09T23:58:35,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/822527172202456dbdddd667ea5c7b72, entries=150, sequenceid=328, filesize=12.0 K 2024-12-09T23:58:35,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/1c8f2c3852d54fc585c1602d18349924 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/1c8f2c3852d54fc585c1602d18349924 2024-12-09T23:58:35,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/1c8f2c3852d54fc585c1602d18349924, entries=150, sequenceid=328, filesize=12.0 K 2024-12-09T23:58:35,037 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for a41ba930d231212908b11d93742f8c4a in 1280ms, sequenceid=328, compaction requested=true 2024-12-09T23:58:35,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a41ba930d231212908b11d93742f8c4a: 2024-12-09T23:58:35,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a41ba930d231212908b11d93742f8c4a:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:58:35,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:35,037 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:58:35,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a41ba930d231212908b11d93742f8c4a:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:58:35,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:35,037 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:58:35,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a41ba930d231212908b11d93742f8c4a:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:58:35,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:58:35,039 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:58:35,039 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:58:35,039 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): a41ba930d231212908b11d93742f8c4a/B is initiating minor compaction (all files) 2024-12-09T23:58:35,039 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): a41ba930d231212908b11d93742f8c4a/A is initiating minor compaction (all files) 2024-12-09T23:58:35,039 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a41ba930d231212908b11d93742f8c4a/B in TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:35,039 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a41ba930d231212908b11d93742f8c4a/A in TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:35,039 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/cd535524bb66440cb8ba280ed327bfad, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/b973bdcf8e1c4c5690999488da6ac585, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/822527172202456dbdddd667ea5c7b72] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp, totalSize=36.6 K 2024-12-09T23:58:35,039 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/5708e58540fa4c75af957e736abfb91b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/1013ac21ff354e79982dd8a3166e766c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/4fecf98e0c48448c87dd67d97c845a46] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp, totalSize=100.6 K 2024-12-09T23:58:35,039 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:35,039 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. files: [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/5708e58540fa4c75af957e736abfb91b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/1013ac21ff354e79982dd8a3166e766c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/4fecf98e0c48448c87dd67d97c845a46] 2024-12-09T23:58:35,039 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting cd535524bb66440cb8ba280ed327bfad, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733788709151 2024-12-09T23:58:35,039 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5708e58540fa4c75af957e736abfb91b, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733788709151 2024-12-09T23:58:35,039 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting b973bdcf8e1c4c5690999488da6ac585, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733788710402 2024-12-09T23:58:35,040 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1013ac21ff354e79982dd8a3166e766c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733788710402 2024-12-09T23:58:35,040 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 822527172202456dbdddd667ea5c7b72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733788712607 2024-12-09T23:58:35,040 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fecf98e0c48448c87dd67d97c845a46, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733788712599 2024-12-09T23:58:35,044 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:35,046 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a41ba930d231212908b11d93742f8c4a#B#compaction#316 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:35,046 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209dd70e42fc3734b7dba004f6c595f3caa_a41ba930d231212908b11d93742f8c4a store=[table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:35,046 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/4d139eb3f3f5480095f2168da4f067f9 is 50, key is test_row_0/B:col10/1733788713755/Put/seqid=0 2024-12-09T23:58:35,047 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209dd70e42fc3734b7dba004f6c595f3caa_a41ba930d231212908b11d93742f8c4a, store=[table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:35,048 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209dd70e42fc3734b7dba004f6c595f3caa_a41ba930d231212908b11d93742f8c4a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:35,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:35,049 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a41ba930d231212908b11d93742f8c4a 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-09T23:58:35,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=A 2024-12-09T23:58:35,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:58:35,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=B 2024-12-09T23:58:35,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:58:35,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=C 2024-12-09T23:58:35,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:58:35,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742201_1377 (size=4469) 2024-12-09T23:58:35,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742200_1376 (size=12949) 2024-12-09T23:58:35,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412093826683409bf431a89e855e86463908a_a41ba930d231212908b11d93742f8c4a is 50, key is test_row_0/A:col10/1733788715048/Put/seqid=0 2024-12-09T23:58:35,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742202_1378 (size=14994) 2024-12-09T23:58:35,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:35,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788775082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:35,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:35,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:35,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:35,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:35,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788775084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:35,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:35,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:35,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:35,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57146 deadline: 1733788775193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:35,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:35,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788775193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:35,197 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_1' in a41ba930d231212908b11d93742f8c4a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:58:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:35,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:35,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788775202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:35,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:35,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:35,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:35,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:35,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57162 deadline: 1733788775213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:35,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:35,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57206 deadline: 1733788775213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:35,220 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_0' in a41ba930d231212908b11d93742f8c4a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8255 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:58:35,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:35,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:35,220 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_1' in a41ba930d231212908b11d93742f8c4a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8257 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:58:35,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:35,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:35,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:35,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:35,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-09T23:58:35,273 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:58:35,273 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:58:35,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-12-09T23:58:35,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-09T23:58:35,274 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:58:35,275 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:58:35,275 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:58:35,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-09T23:58:35,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:35,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788775402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:35,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:35,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:35,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:35,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:35,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788775413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:35,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:35,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:35,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:35,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-09T23:58:35,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:35,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:35,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:35,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:35,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:35,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:35,462 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a41ba930d231212908b11d93742f8c4a#A#compaction#315 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:35,462 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/11c1a7c382e44cd3b9c6affc4c8db280 is 175, key is test_row_0/A:col10/1733788713755/Put/seqid=0 2024-12-09T23:58:35,465 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/4d139eb3f3f5480095f2168da4f067f9 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/4d139eb3f3f5480095f2168da4f067f9 2024-12-09T23:58:35,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742203_1379 (size=31903) 2024-12-09T23:58:35,474 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a41ba930d231212908b11d93742f8c4a/B of a41ba930d231212908b11d93742f8c4a into 4d139eb3f3f5480095f2168da4f067f9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:35,474 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/11c1a7c382e44cd3b9c6affc4c8db280 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/11c1a7c382e44cd3b9c6affc4c8db280 2024-12-09T23:58:35,474 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a41ba930d231212908b11d93742f8c4a: 2024-12-09T23:58:35,474 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., storeName=a41ba930d231212908b11d93742f8c4a/B, priority=13, startTime=1733788715037; duration=0sec 2024-12-09T23:58:35,474 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:58:35,474 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a41ba930d231212908b11d93742f8c4a:B 2024-12-09T23:58:35,475 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:58:35,477 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:58:35,477 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): a41ba930d231212908b11d93742f8c4a/C is initiating minor compaction (all files) 2024-12-09T23:58:35,477 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a41ba930d231212908b11d93742f8c4a/C in TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:35,477 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/862b21811cdd4d0fa931d2a40339821c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/84b238a61a8441798c9c45f313ea06d9, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/1c8f2c3852d54fc585c1602d18349924] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp, totalSize=36.6 K 2024-12-09T23:58:35,478 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 862b21811cdd4d0fa931d2a40339821c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733788709151 2024-12-09T23:58:35,478 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 84b238a61a8441798c9c45f313ea06d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733788710402 2024-12-09T23:58:35,478 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c8f2c3852d54fc585c1602d18349924, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733788712607 2024-12-09T23:58:35,480 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:58:35,480 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a41ba930d231212908b11d93742f8c4a/A of a41ba930d231212908b11d93742f8c4a into 11c1a7c382e44cd3b9c6affc4c8db280(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:35,480 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a41ba930d231212908b11d93742f8c4a: 2024-12-09T23:58:35,480 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., storeName=a41ba930d231212908b11d93742f8c4a/A, priority=13, startTime=1733788715037; duration=0sec 2024-12-09T23:58:35,480 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:35,480 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a41ba930d231212908b11d93742f8c4a:A 2024-12-09T23:58:35,484 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412093826683409bf431a89e855e86463908a_a41ba930d231212908b11d93742f8c4a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412093826683409bf431a89e855e86463908a_a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:35,484 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/62745928f5bf4dcfb93d747a52d78c4d, store: [table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:35,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/62745928f5bf4dcfb93d747a52d78c4d is 175, key is test_row_0/A:col10/1733788715048/Put/seqid=0 2024-12-09T23:58:35,485 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a41ba930d231212908b11d93742f8c4a#C#compaction#318 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:35,485 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/b041a81f42154c0bbe14b7b43d5bbf28 is 50, key is test_row_0/C:col10/1733788713755/Put/seqid=0 2024-12-09T23:58:35,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742204_1380 (size=39949) 2024-12-09T23:58:35,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742205_1381 (size=12949) 2024-12-09T23:58:35,501 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/b041a81f42154c0bbe14b7b43d5bbf28 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/b041a81f42154c0bbe14b7b43d5bbf28 2024-12-09T23:58:35,504 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a41ba930d231212908b11d93742f8c4a/C of a41ba930d231212908b11d93742f8c4a into b041a81f42154c0bbe14b7b43d5bbf28(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:35,505 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a41ba930d231212908b11d93742f8c4a: 2024-12-09T23:58:35,505 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., storeName=a41ba930d231212908b11d93742f8c4a/C, priority=13, startTime=1733788715037; duration=0sec 2024-12-09T23:58:35,505 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:35,505 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a41ba930d231212908b11d93742f8c4a:C 2024-12-09T23:58:35,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-09T23:58:35,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:35,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:35,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:35,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:35,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:35,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:35,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-09T23:58:35,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:35,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788775713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:35,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:35,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:35,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:35,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788775723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:35,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:35,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:35,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:35,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-09T23:58:35,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:35,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:35,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:35,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:35,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:35,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:35,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-09T23:58:35,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:35,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:35,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:35,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:35,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:35,889 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=364, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/62745928f5bf4dcfb93d747a52d78c4d 2024-12-09T23:58:35,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/fcae2539cab0470a934a663ab281a6c9 is 50, key is test_row_0/B:col10/1733788715048/Put/seqid=0 2024-12-09T23:58:35,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742206_1382 (size=12301) 2024-12-09T23:58:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-09T23:58:36,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-09T23:58:36,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:36,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:36,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:36,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:36,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:36,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:36,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-09T23:58:36,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:36,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:36,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:36,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:36,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:36,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:36,218 INFO [Thread-1399 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1399. 2024-12-09T23:58:36,218 DEBUG [Thread-1399 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:58:36,218 DEBUG [Thread-1399 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:36,218 DEBUG [Thread-1399 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:36,218 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:58:36,219 INFO [Thread-1401 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1401. 2024-12-09T23:58:36,219 DEBUG [Thread-1401 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:58:36,219 DEBUG [Thread-1401 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:36,219 DEBUG [Thread-1401 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:36,219 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:58:36,221 INFO [Thread-1403 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1403. 2024-12-09T23:58:36,221 DEBUG [Thread-1403 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:58:36,221 DEBUG [Thread-1403 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:36,221 DEBUG [Thread-1403 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:36,221 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:58:36,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:36,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57188 deadline: 1733788776223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:36,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:36,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:36,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:36,225 INFO [Thread-1397 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1397. 2024-12-09T23:58:36,225 DEBUG [Thread-1397 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:58:36,225 DEBUG [Thread-1397 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:36,225 DEBUG [Thread-1397 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:36,225 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:58:36,226 INFO [Thread-1395 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1395. 2024-12-09T23:58:36,226 DEBUG [Thread-1395 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:58:36,226 DEBUG [Thread-1395 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:36,226 DEBUG [Thread-1395 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:36,226 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:58:36,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:36,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57186 deadline: 1733788776233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:36,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:36,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:36,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:58:36,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/fcae2539cab0470a934a663ab281a6c9 2024-12-09T23:58:36,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/8c2ccad7424e44d598c5d3f5bdcbca96 is 50, key is test_row_0/C:col10/1733788715048/Put/seqid=0 2024-12-09T23:58:36,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742207_1383 (size=12301) 2024-12-09T23:58:36,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-09T23:58:36,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:36,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:36,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:36,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:36,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:36,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:36,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-09T23:58:36,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-09T23:58:36,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:36,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:36,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:36,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:36,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:36,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-09T23:58:36,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:36,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:36,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:36,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:36,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:36,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:36,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/8c2ccad7424e44d598c5d3f5bdcbca96 2024-12-09T23:58:36,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/62745928f5bf4dcfb93d747a52d78c4d as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/62745928f5bf4dcfb93d747a52d78c4d 2024-12-09T23:58:36,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/62745928f5bf4dcfb93d747a52d78c4d, entries=200, sequenceid=364, filesize=39.0 K 2024-12-09T23:58:36,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/fcae2539cab0470a934a663ab281a6c9 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/fcae2539cab0470a934a663ab281a6c9 2024-12-09T23:58:36,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/fcae2539cab0470a934a663ab281a6c9, entries=150, sequenceid=364, filesize=12.0 K 2024-12-09T23:58:36,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/8c2ccad7424e44d598c5d3f5bdcbca96 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/8c2ccad7424e44d598c5d3f5bdcbca96 2024-12-09T23:58:36,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/8c2ccad7424e44d598c5d3f5bdcbca96, entries=150, sequenceid=364, filesize=12.0 K 2024-12-09T23:58:36,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for a41ba930d231212908b11d93742f8c4a in 1691ms, sequenceid=364, compaction requested=false 2024-12-09T23:58:36,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a41ba930d231212908b11d93742f8c4a: 2024-12-09T23:58:36,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-09T23:58:36,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:36,813 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing a41ba930d231212908b11d93742f8c4a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T23:58:36,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=A 2024-12-09T23:58:36,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:36,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=B 2024-12-09T23:58:36,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:36,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=C 2024-12-09T23:58:36,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:36,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209aa880620aa714e0daae8807bc6085469_a41ba930d231212908b11d93742f8c4a is 50, key is test_row_0/A:col10/1733788715081/Put/seqid=0 2024-12-09T23:58:36,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742208_1384 (size=12454) 2024-12-09T23:58:37,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:58:37,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:37,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. as already flushing 2024-12-09T23:58:37,236 INFO [Thread-1386 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1386. 2024-12-09T23:58:37,236 DEBUG [Thread-1386 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:58:37,237 DEBUG [Thread-1386 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:37,237 DEBUG [Thread-1386 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:37,237 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:58:37,239 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209aa880620aa714e0daae8807bc6085469_a41ba930d231212908b11d93742f8c4a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209aa880620aa714e0daae8807bc6085469_a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:37,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/c7ffbb3bfba34970afa3ec0ce8b472a5, store: [table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:37,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/c7ffbb3bfba34970afa3ec0ce8b472a5 is 175, key is test_row_0/A:col10/1733788715081/Put/seqid=0 2024-12-09T23:58:37,245 INFO [Thread-1392 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1392. 2024-12-09T23:58:37,245 DEBUG [Thread-1392 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:58:37,245 DEBUG [Thread-1392 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:37,245 DEBUG [Thread-1392 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:37,246 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:58:37,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742209_1385 (size=31255) 2024-12-09T23:58:37,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-09T23:58:37,649 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=377, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/c7ffbb3bfba34970afa3ec0ce8b472a5 2024-12-09T23:58:37,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/43d7c675b588433fafdd3af5d99b1271 is 50, key is test_row_0/B:col10/1733788715081/Put/seqid=0 2024-12-09T23:58:37,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742210_1386 (size=12301) 2024-12-09T23:58:38,068 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/43d7c675b588433fafdd3af5d99b1271 2024-12-09T23:58:38,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/4f5f5f770c8541289f30e9654a616fa4 is 50, key is test_row_0/C:col10/1733788715081/Put/seqid=0 2024-12-09T23:58:38,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742211_1387 (size=12301) 2024-12-09T23:58:38,292 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T23:58:38,485 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/4f5f5f770c8541289f30e9654a616fa4 2024-12-09T23:58:38,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/c7ffbb3bfba34970afa3ec0ce8b472a5 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/c7ffbb3bfba34970afa3ec0ce8b472a5 2024-12-09T23:58:38,499 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/c7ffbb3bfba34970afa3ec0ce8b472a5, entries=150, sequenceid=377, filesize=30.5 K 2024-12-09T23:58:38,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/43d7c675b588433fafdd3af5d99b1271 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/43d7c675b588433fafdd3af5d99b1271 2024-12-09T23:58:38,505 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/43d7c675b588433fafdd3af5d99b1271, entries=150, sequenceid=377, filesize=12.0 K 2024-12-09T23:58:38,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/4f5f5f770c8541289f30e9654a616fa4 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/4f5f5f770c8541289f30e9654a616fa4 2024-12-09T23:58:38,509 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/4f5f5f770c8541289f30e9654a616fa4, entries=150, sequenceid=377, filesize=12.0 K 2024-12-09T23:58:38,510 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=13.42 KB/13740 for a41ba930d231212908b11d93742f8c4a in 1697ms, sequenceid=377, compaction requested=true 2024-12-09T23:58:38,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for a41ba930d231212908b11d93742f8c4a: 2024-12-09T23:58:38,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:38,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-12-09T23:58:38,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-12-09T23:58:38,512 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-09T23:58:38,512 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2360 sec 2024-12-09T23:58:38,513 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 3.2400 sec 2024-12-09T23:58:39,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-09T23:58:39,444 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:58:42,859 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2907d75fbb3e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/113ccf98314b40feb5fc031a77d349d0, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/e04d38025de341159020218d51fae15d, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/1c3cc55643874b41b4b0119cb7273d05, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/a59d70e4ae6e45339889b8bd957d2ff0, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/9106e0fb57a54898a2c362c788e1493f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/1d78d53a064649cc8028e98f44bd2e1c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/0a00ecb419404fdd89b2f8fbcc51f5f6, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/9b489fa75fd54d0da70367020c43f08c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/0cf9439c07204f7b9c8fae1a3d8d00bd, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/07d5ce375e6e4782ac819d9bc49af226, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/f4cae3fd02b6439e8ac312f76033a823, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/c8f581ca85f04637a4cee0af1fe0f15b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/2f45f45dcc5c4f78b2c38320b81ec3cb, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/783827b3fb9b4f63bd297a660d957c27, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/83379ef2c2db4dd5a3ef09505b639c78, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/aed34e7e81224f079ce0fd98293cb91e, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/5708e58540fa4c75af957e736abfb91b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/1013ac21ff354e79982dd8a3166e766c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/4fecf98e0c48448c87dd67d97c845a46] to archive 2024-12-09T23:58:42,860 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2907d75fbb3e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-09T23:58:42,863 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/113ccf98314b40feb5fc031a77d349d0 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/113ccf98314b40feb5fc031a77d349d0 2024-12-09T23:58:42,863 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/1c3cc55643874b41b4b0119cb7273d05 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/1c3cc55643874b41b4b0119cb7273d05 2024-12-09T23:58:42,863 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/e04d38025de341159020218d51fae15d to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/e04d38025de341159020218d51fae15d 2024-12-09T23:58:42,863 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/a59d70e4ae6e45339889b8bd957d2ff0 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/a59d70e4ae6e45339889b8bd957d2ff0 2024-12-09T23:58:42,863 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/9106e0fb57a54898a2c362c788e1493f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/9106e0fb57a54898a2c362c788e1493f 2024-12-09T23:58:42,863 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/1d78d53a064649cc8028e98f44bd2e1c to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/1d78d53a064649cc8028e98f44bd2e1c 2024-12-09T23:58:42,863 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/0a00ecb419404fdd89b2f8fbcc51f5f6 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/0a00ecb419404fdd89b2f8fbcc51f5f6 2024-12-09T23:58:42,863 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/9b489fa75fd54d0da70367020c43f08c to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/9b489fa75fd54d0da70367020c43f08c 2024-12-09T23:58:42,865 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/783827b3fb9b4f63bd297a660d957c27 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/783827b3fb9b4f63bd297a660d957c27 2024-12-09T23:58:42,865 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/2f45f45dcc5c4f78b2c38320b81ec3cb to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/2f45f45dcc5c4f78b2c38320b81ec3cb 2024-12-09T23:58:42,865 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/c8f581ca85f04637a4cee0af1fe0f15b to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/c8f581ca85f04637a4cee0af1fe0f15b 2024-12-09T23:58:42,865 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/07d5ce375e6e4782ac819d9bc49af226 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/07d5ce375e6e4782ac819d9bc49af226 2024-12-09T23:58:42,866 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/0cf9439c07204f7b9c8fae1a3d8d00bd to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/0cf9439c07204f7b9c8fae1a3d8d00bd 2024-12-09T23:58:42,866 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/f4cae3fd02b6439e8ac312f76033a823 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/f4cae3fd02b6439e8ac312f76033a823 2024-12-09T23:58:42,866 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/83379ef2c2db4dd5a3ef09505b639c78 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/83379ef2c2db4dd5a3ef09505b639c78 2024-12-09T23:58:42,866 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/aed34e7e81224f079ce0fd98293cb91e to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/aed34e7e81224f079ce0fd98293cb91e 2024-12-09T23:58:42,866 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/5708e58540fa4c75af957e736abfb91b to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/5708e58540fa4c75af957e736abfb91b 2024-12-09T23:58:42,867 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/1013ac21ff354e79982dd8a3166e766c to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/1013ac21ff354e79982dd8a3166e766c 2024-12-09T23:58:42,867 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/4fecf98e0c48448c87dd67d97c845a46 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/4fecf98e0c48448c87dd67d97c845a46 2024-12-09T23:58:42,872 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2907d75fbb3e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/657d57f7b27c440ca69fec4f185c98ac, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/db8290db3c524ff1b3afb72e6898e6b8, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/9f84e5c505754ca4b34c6463ff9446e9, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/d60aedb81d014d159566b200026308c4, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/82fd67353853480b840cfa1fb1623cf1, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/f7f1b6057e3148a8b4884860647d5e69, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/9e54be9e153f4fb2ad95f0bc6eb15fca, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/b6e7fbd7484f42b287024c4d378d9f0f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/4c554236fbb04c48a2b2a0de3346915c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/4311dafb6a974a839c116696680221ea, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/2513aa2dc59f41c5a3078738d569457c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/7b389894d9e642d497646c2a9ba20e8b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/63bd34ea453d4202890328433fa9a8ad, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/04e99d97650c4254be90667eb31f4ec2, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/074285c26b2f4232bb9e5a1744fc1dcc, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/cd535524bb66440cb8ba280ed327bfad, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/26996ff0b1b74aac9bf8d06916b9880b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/b973bdcf8e1c4c5690999488da6ac585, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/822527172202456dbdddd667ea5c7b72] to archive 2024-12-09T23:58:42,872 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2907d75fbb3e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-09T23:58:42,874 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/657d57f7b27c440ca69fec4f185c98ac to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/657d57f7b27c440ca69fec4f185c98ac 2024-12-09T23:58:42,874 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/db8290db3c524ff1b3afb72e6898e6b8 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/db8290db3c524ff1b3afb72e6898e6b8 2024-12-09T23:58:42,874 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/9f84e5c505754ca4b34c6463ff9446e9 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/9f84e5c505754ca4b34c6463ff9446e9 2024-12-09T23:58:42,874 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/82fd67353853480b840cfa1fb1623cf1 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/82fd67353853480b840cfa1fb1623cf1 2024-12-09T23:58:42,874 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/d60aedb81d014d159566b200026308c4 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/d60aedb81d014d159566b200026308c4 2024-12-09T23:58:42,874 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/f7f1b6057e3148a8b4884860647d5e69 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/f7f1b6057e3148a8b4884860647d5e69 2024-12-09T23:58:42,874 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/9e54be9e153f4fb2ad95f0bc6eb15fca to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/9e54be9e153f4fb2ad95f0bc6eb15fca 2024-12-09T23:58:42,874 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/b6e7fbd7484f42b287024c4d378d9f0f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/b6e7fbd7484f42b287024c4d378d9f0f 2024-12-09T23:58:42,875 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/4c554236fbb04c48a2b2a0de3346915c to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/4c554236fbb04c48a2b2a0de3346915c 2024-12-09T23:58:42,875 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/7b389894d9e642d497646c2a9ba20e8b to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/7b389894d9e642d497646c2a9ba20e8b 2024-12-09T23:58:42,875 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/2513aa2dc59f41c5a3078738d569457c to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/2513aa2dc59f41c5a3078738d569457c 2024-12-09T23:58:42,875 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/4311dafb6a974a839c116696680221ea to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/4311dafb6a974a839c116696680221ea 2024-12-09T23:58:42,875 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/cd535524bb66440cb8ba280ed327bfad to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/cd535524bb66440cb8ba280ed327bfad 2024-12-09T23:58:42,875 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/63bd34ea453d4202890328433fa9a8ad to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/63bd34ea453d4202890328433fa9a8ad 2024-12-09T23:58:42,875 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/04e99d97650c4254be90667eb31f4ec2 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/04e99d97650c4254be90667eb31f4ec2 2024-12-09T23:58:42,875 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/074285c26b2f4232bb9e5a1744fc1dcc to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/074285c26b2f4232bb9e5a1744fc1dcc 2024-12-09T23:58:42,876 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/b973bdcf8e1c4c5690999488da6ac585 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/b973bdcf8e1c4c5690999488da6ac585 2024-12-09T23:58:42,876 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/26996ff0b1b74aac9bf8d06916b9880b to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/26996ff0b1b74aac9bf8d06916b9880b 2024-12-09T23:58:42,876 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/822527172202456dbdddd667ea5c7b72 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/822527172202456dbdddd667ea5c7b72 2024-12-09T23:58:42,879 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2907d75fbb3e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/9c5369322ec34e07bc9c4dc8d7fa4f48, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/171d5358f36845ceafbf8fb213a60450, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/23401302d1214fe894c887a436c15f21, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/d5833ebcdb9f4cc696c71bfd7898eed8, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/c299cd83d09b4228934170feca6bc897, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/c6530ec49e1d4ad595bfb9f16e3d096c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/a2955c9d85ce448dbe0701d6de7c4994, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/cc8cdda963204f42a6c7b3444f6adf19, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/02c6e5e5657f43a9a4c2aaeeca0c688b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/0037fbbb61ed4a2287ecb1189cbe4c14, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/321f3a5fb4f341db98854322e9e0d482, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/22d640ec7f6f443f82df260ffa71f73b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/b998699045894360955c2a4337f830b6, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/cb779f9c7dc340c5bdf9131e11468d60, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/4de0bf0dfcec44a89a9b64a351ae3f6a, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/862b21811cdd4d0fa931d2a40339821c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/353db2e1ba054776adc3b178b78cf683, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/84b238a61a8441798c9c45f313ea06d9, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/1c8f2c3852d54fc585c1602d18349924] to archive 2024-12-09T23:58:42,880 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2907d75fbb3e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-09T23:58:42,881 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/d5833ebcdb9f4cc696c71bfd7898eed8 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/d5833ebcdb9f4cc696c71bfd7898eed8 2024-12-09T23:58:42,882 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/171d5358f36845ceafbf8fb213a60450 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/171d5358f36845ceafbf8fb213a60450 2024-12-09T23:58:42,882 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/9c5369322ec34e07bc9c4dc8d7fa4f48 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/9c5369322ec34e07bc9c4dc8d7fa4f48 2024-12-09T23:58:42,882 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/23401302d1214fe894c887a436c15f21 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/23401302d1214fe894c887a436c15f21 2024-12-09T23:58:42,882 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/a2955c9d85ce448dbe0701d6de7c4994 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/a2955c9d85ce448dbe0701d6de7c4994 2024-12-09T23:58:42,882 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/c6530ec49e1d4ad595bfb9f16e3d096c to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/c6530ec49e1d4ad595bfb9f16e3d096c 2024-12-09T23:58:42,882 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/cc8cdda963204f42a6c7b3444f6adf19 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/cc8cdda963204f42a6c7b3444f6adf19 2024-12-09T23:58:42,882 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/c299cd83d09b4228934170feca6bc897 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/c299cd83d09b4228934170feca6bc897 2024-12-09T23:58:42,883 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/02c6e5e5657f43a9a4c2aaeeca0c688b to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/02c6e5e5657f43a9a4c2aaeeca0c688b 2024-12-09T23:58:42,883 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/0037fbbb61ed4a2287ecb1189cbe4c14 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/0037fbbb61ed4a2287ecb1189cbe4c14 2024-12-09T23:58:42,883 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/b998699045894360955c2a4337f830b6 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/b998699045894360955c2a4337f830b6 2024-12-09T23:58:42,883 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/cb779f9c7dc340c5bdf9131e11468d60 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/cb779f9c7dc340c5bdf9131e11468d60 2024-12-09T23:58:42,883 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/4de0bf0dfcec44a89a9b64a351ae3f6a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/4de0bf0dfcec44a89a9b64a351ae3f6a 2024-12-09T23:58:42,884 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/22d640ec7f6f443f82df260ffa71f73b to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/22d640ec7f6f443f82df260ffa71f73b 2024-12-09T23:58:42,884 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/321f3a5fb4f341db98854322e9e0d482 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/321f3a5fb4f341db98854322e9e0d482 2024-12-09T23:58:42,884 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/862b21811cdd4d0fa931d2a40339821c to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/862b21811cdd4d0fa931d2a40339821c 2024-12-09T23:58:42,884 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/353db2e1ba054776adc3b178b78cf683 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/353db2e1ba054776adc3b178b78cf683 2024-12-09T23:58:42,884 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/84b238a61a8441798c9c45f313ea06d9 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/84b238a61a8441798c9c45f313ea06d9 2024-12-09T23:58:42,884 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/1c8f2c3852d54fc585c1602d18349924 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/1c8f2c3852d54fc585c1602d18349924 2024-12-09T23:58:45,258 INFO [Thread-1384 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1384. 2024-12-09T23:58:45,258 INFO [Thread-1390 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1390. 2024-12-09T23:58:45,258 DEBUG [Thread-1384 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:58:45,258 DEBUG [Thread-1390 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:58:45,258 DEBUG [Thread-1384 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:45,258 DEBUG [Thread-1390 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:45,258 DEBUG [Thread-1390 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:45,258 DEBUG [Thread-1384 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:45,258 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:58:45,258 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:58:45,306 INFO [Thread-1388 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1388. 2024-12-09T23:58:45,306 DEBUG [Thread-1388 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:58:45,306 DEBUG [Thread-1388 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:45,306 DEBUG [Thread-1388 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:45,306 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:58:45,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T23:58:45,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-12-09T23:58:45,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 99 2024-12-09T23:58:45,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-12-09T23:58:45,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-12-09T23:58:45,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-12-09T23:58:45,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T23:58:45,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T23:58:45,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1689 2024-12-09T23:58:45,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5037 rows 2024-12-09T23:58:45,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1683 2024-12-09T23:58:45,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5031 rows 2024-12-09T23:58:45,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1684 2024-12-09T23:58:45,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5031 rows 2024-12-09T23:58:45,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1677 2024-12-09T23:58:45,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5025 rows 2024-12-09T23:58:45,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1678 2024-12-09T23:58:45,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5031 rows 2024-12-09T23:58:45,307 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T23:58:45,307 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T23:58:45,308 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:45,308 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:45,308 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:58:45,309 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-09T23:58:45,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T23:58:45,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-09T23:58:45,315 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733788725315"}]},"ts":"1733788725315"} 2024-12-09T23:58:45,317 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T23:58:45,317 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T23:58:45,318 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T23:58:45,320 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a41ba930d231212908b11d93742f8c4a, UNASSIGN}] 2024-12-09T23:58:45,321 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a41ba930d231212908b11d93742f8c4a, UNASSIGN 2024-12-09T23:58:45,322 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=a41ba930d231212908b11d93742f8c4a, regionState=CLOSING, regionLocation=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:45,323 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a41ba930d231212908b11d93742f8c4a, UNASSIGN because future has completed 2024-12-09T23:58:45,323 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T23:58:45,323 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837}] 2024-12-09T23:58:45,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-09T23:58:45,476 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(122): Close a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:45,476 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T23:58:45,476 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1722): Closing a41ba930d231212908b11d93742f8c4a, disabling compactions & flushes 2024-12-09T23:58:45,476 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:45,476 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:45,476 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. after waiting 0 ms 2024-12-09T23:58:45,476 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:45,476 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(2902): Flushing a41ba930d231212908b11d93742f8c4a 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-09T23:58:45,476 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=A 2024-12-09T23:58:45,476 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:45,476 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=B 2024-12-09T23:58:45,476 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:45,476 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a41ba930d231212908b11d93742f8c4a, store=C 2024-12-09T23:58:45,476 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:45,481 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209b1b38474880b46a5999fd20099f09695_a41ba930d231212908b11d93742f8c4a is 50, key is test_row_0/A:col10/1733788725254/Put/seqid=0 2024-12-09T23:58:45,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742212_1388 (size=9914) 2024-12-09T23:58:45,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-09T23:58:45,886 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:58:45,897 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209b1b38474880b46a5999fd20099f09695_a41ba930d231212908b11d93742f8c4a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b1b38474880b46a5999fd20099f09695_a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:45,898 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/a6be57a8e584498ca931081e39542ceb, store: [table=TestAcidGuarantees family=A region=a41ba930d231212908b11d93742f8c4a] 2024-12-09T23:58:45,899 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/a6be57a8e584498ca931081e39542ceb is 175, key is test_row_0/A:col10/1733788725254/Put/seqid=0 2024-12-09T23:58:45,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742213_1389 (size=22561) 2024-12-09T23:58:45,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-09T23:58:46,305 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=385, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/a6be57a8e584498ca931081e39542ceb 2024-12-09T23:58:46,318 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/485bad7b9bde4136942fc31bea00962c is 50, key is test_row_0/B:col10/1733788725254/Put/seqid=0 2024-12-09T23:58:46,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742214_1390 (size=9857) 2024-12-09T23:58:46,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-09T23:58:46,724 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/485bad7b9bde4136942fc31bea00962c 2024-12-09T23:58:46,735 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/150090fdd5a04928ae7ad064424aac9d is 50, key is test_row_0/C:col10/1733788725254/Put/seqid=0 2024-12-09T23:58:46,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742215_1391 (size=9857) 2024-12-09T23:58:47,141 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/150090fdd5a04928ae7ad064424aac9d 2024-12-09T23:58:47,196 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/A/a6be57a8e584498ca931081e39542ceb as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/a6be57a8e584498ca931081e39542ceb 2024-12-09T23:58:47,202 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/a6be57a8e584498ca931081e39542ceb, entries=100, sequenceid=385, filesize=22.0 K 2024-12-09T23:58:47,203 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/B/485bad7b9bde4136942fc31bea00962c as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/485bad7b9bde4136942fc31bea00962c 2024-12-09T23:58:47,207 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/485bad7b9bde4136942fc31bea00962c, entries=100, sequenceid=385, filesize=9.6 K 2024-12-09T23:58:47,208 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/.tmp/C/150090fdd5a04928ae7ad064424aac9d as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/150090fdd5a04928ae7ad064424aac9d 2024-12-09T23:58:47,213 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/150090fdd5a04928ae7ad064424aac9d, entries=100, sequenceid=385, filesize=9.6 K 2024-12-09T23:58:47,214 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for a41ba930d231212908b11d93742f8c4a in 1738ms, sequenceid=385, compaction requested=true 2024-12-09T23:58:47,218 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/recovered.edits/388.seqid, newMaxSeqId=388, maxSeqId=4 2024-12-09T23:58:47,218 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. 2024-12-09T23:58:47,218 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1676): Region close journal for a41ba930d231212908b11d93742f8c4a: Waiting for close lock at 1733788725476Running coprocessor pre-close hooks at 1733788725476Disabling compacts and flushes for region at 1733788725476Disabling writes for close at 1733788725476Obtaining lock to block concurrent updates at 1733788725476Preparing flush snapshotting stores in a41ba930d231212908b11d93742f8c4a at 1733788725476Finished memstore snapshotting TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733788725476Flushing stores of TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a. at 1733788725477 (+1 ms)Flushing a41ba930d231212908b11d93742f8c4a/A: creating writer at 1733788725477Flushing mob file a41ba930d231212908b11d93742f8c4a/A: appending metadata at 1733788725481 (+4 ms)Flushing mob file a41ba930d231212908b11d93742f8c4a/A: closing flushed file at 1733788725481Flushing a41ba930d231212908b11d93742f8c4a/A: appending metadata at 1733788725898 (+417 ms)Flushing a41ba930d231212908b11d93742f8c4a/A: closing flushed file at 1733788725898Flushing a41ba930d231212908b11d93742f8c4a/B: creating writer at 1733788726315 (+417 ms)Flushing a41ba930d231212908b11d93742f8c4a/B: appending metadata at 1733788726318 (+3 ms)Flushing a41ba930d231212908b11d93742f8c4a/B: closing flushed file at 1733788726318Flushing a41ba930d231212908b11d93742f8c4a/C: creating writer at 1733788726731 (+413 ms)Flushing a41ba930d231212908b11d93742f8c4a/C: appending metadata at 1733788726734 (+3 ms)Flushing a41ba930d231212908b11d93742f8c4a/C: closing flushed file at 1733788726734Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4fdc003c: reopening flushed file at 1733788727194 (+460 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5bdba6cd: reopening flushed file at 1733788727202 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@690aecbb: reopening flushed file at 1733788727207 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for a41ba930d231212908b11d93742f8c4a in 1738ms, sequenceid=385, compaction requested=true at 1733788727214 (+7 ms)Writing region close event to WAL at 1733788727215 (+1 ms)Running coprocessor post-close hooks at 1733788727218 (+3 ms)Closed at 1733788727218 2024-12-09T23:58:47,220 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(157): Closed a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:47,220 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=a41ba930d231212908b11d93742f8c4a, regionState=CLOSED 2024-12-09T23:58:47,221 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 because future has completed 2024-12-09T23:58:47,223 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-09T23:58:47,223 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseRegionProcedure a41ba930d231212908b11d93742f8c4a, server=2907d75fbb3e,37627,1733788599837 in 1.8990 sec 2024-12-09T23:58:47,225 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-12-09T23:58:47,225 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a41ba930d231212908b11d93742f8c4a, UNASSIGN in 1.9030 sec 2024-12-09T23:58:47,226 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-09T23:58:47,226 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9070 sec 2024-12-09T23:58:47,227 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733788727227"}]},"ts":"1733788727227"} 2024-12-09T23:58:47,229 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T23:58:47,229 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T23:58:47,230 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9200 sec 2024-12-09T23:58:47,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-09T23:58:47,474 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:58:47,476 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-09T23:58:47,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T23:58:47,479 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T23:58:47,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-09T23:58:47,480 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=111, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T23:58:47,483 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:47,487 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A, FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B, FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C, FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/recovered.edits] 2024-12-09T23:58:47,493 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/11c1a7c382e44cd3b9c6affc4c8db280 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/11c1a7c382e44cd3b9c6affc4c8db280 2024-12-09T23:58:47,493 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/62745928f5bf4dcfb93d747a52d78c4d to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/62745928f5bf4dcfb93d747a52d78c4d 2024-12-09T23:58:47,493 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/a6be57a8e584498ca931081e39542ceb to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/a6be57a8e584498ca931081e39542ceb 2024-12-09T23:58:47,493 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/c7ffbb3bfba34970afa3ec0ce8b472a5 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/A/c7ffbb3bfba34970afa3ec0ce8b472a5 2024-12-09T23:58:47,498 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/43d7c675b588433fafdd3af5d99b1271 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/43d7c675b588433fafdd3af5d99b1271 2024-12-09T23:58:47,499 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/485bad7b9bde4136942fc31bea00962c to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/485bad7b9bde4136942fc31bea00962c 2024-12-09T23:58:47,499 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/4d139eb3f3f5480095f2168da4f067f9 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/4d139eb3f3f5480095f2168da4f067f9 2024-12-09T23:58:47,499 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/fcae2539cab0470a934a663ab281a6c9 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/B/fcae2539cab0470a934a663ab281a6c9 2024-12-09T23:58:47,503 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/150090fdd5a04928ae7ad064424aac9d to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/150090fdd5a04928ae7ad064424aac9d 2024-12-09T23:58:47,503 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/4f5f5f770c8541289f30e9654a616fa4 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/4f5f5f770c8541289f30e9654a616fa4 2024-12-09T23:58:47,503 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/8c2ccad7424e44d598c5d3f5bdcbca96 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/8c2ccad7424e44d598c5d3f5bdcbca96 2024-12-09T23:58:47,504 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/b041a81f42154c0bbe14b7b43d5bbf28 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/C/b041a81f42154c0bbe14b7b43d5bbf28 2024-12-09T23:58:47,507 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/recovered.edits/388.seqid to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a/recovered.edits/388.seqid 2024-12-09T23:58:47,508 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:47,508 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T23:58:47,509 DEBUG [PEWorker-3 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T23:58:47,510 DEBUG [PEWorker-3 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-09T23:58:47,520 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209040435876ade4e1785f1c0bffc3bfcee_a41ba930d231212908b11d93742f8c4a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209040435876ade4e1785f1c0bffc3bfcee_a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:47,520 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209124370a7224946a281a3df7bce8cc830_a41ba930d231212908b11d93742f8c4a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209124370a7224946a281a3df7bce8cc830_a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:47,520 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120911442a420a28452f9db0c3c1caed928e_a41ba930d231212908b11d93742f8c4a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120911442a420a28452f9db0c3c1caed928e_a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:47,520 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092afb060746f849caab6229f5f8ccfd96_a41ba930d231212908b11d93742f8c4a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092afb060746f849caab6229f5f8ccfd96_a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:47,520 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209378611ce5a8d47c5a2940c86f302271f_a41ba930d231212908b11d93742f8c4a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209378611ce5a8d47c5a2940c86f302271f_a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:47,521 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412093826683409bf431a89e855e86463908a_a41ba930d231212908b11d93742f8c4a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412093826683409bf431a89e855e86463908a_a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:47,521 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209664ccd3a7ee541b5987f5c53a92cd609_a41ba930d231212908b11d93742f8c4a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209664ccd3a7ee541b5987f5c53a92cd609_a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:47,521 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412095f8d08ef2d994eca98dc0e660bdd2dd4_a41ba930d231212908b11d93742f8c4a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412095f8d08ef2d994eca98dc0e660bdd2dd4_a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:47,522 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120986ac137acce449a38560fe816f326130_a41ba930d231212908b11d93742f8c4a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120986ac137acce449a38560fe816f326130_a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:47,522 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099d3018ea13774997a8b0bb171b118631_a41ba930d231212908b11d93742f8c4a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099d3018ea13774997a8b0bb171b118631_a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:47,522 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b1b38474880b46a5999fd20099f09695_a41ba930d231212908b11d93742f8c4a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b1b38474880b46a5999fd20099f09695_a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:47,522 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ab70d53c84fe4d92b82ec410afc4e655_a41ba930d231212908b11d93742f8c4a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ab70d53c84fe4d92b82ec410afc4e655_a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:47,522 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209aa880620aa714e0daae8807bc6085469_a41ba930d231212908b11d93742f8c4a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209aa880620aa714e0daae8807bc6085469_a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:47,523 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a15ad80f563142479033c79e9e066670_a41ba930d231212908b11d93742f8c4a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a15ad80f563142479033c79e9e066670_a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:47,523 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d4d5cd7c8b3d4955b173f42ddb6d2851_a41ba930d231212908b11d93742f8c4a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d4d5cd7c8b3d4955b173f42ddb6d2851_a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:47,523 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e0ff9dd8df654f92a3fbb03ea5d6c98b_a41ba930d231212908b11d93742f8c4a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e0ff9dd8df654f92a3fbb03ea5d6c98b_a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:47,523 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ea14a8f17e714226b0b93f49f3f90925_a41ba930d231212908b11d93742f8c4a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ea14a8f17e714226b0b93f49f3f90925_a41ba930d231212908b11d93742f8c4a 2024-12-09T23:58:47,524 DEBUG [PEWorker-3 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T23:58:47,526 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=111, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T23:58:47,528 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T23:58:47,531 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T23:58:47,532 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=111, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T23:58:47,532 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-09T23:58:47,532 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733788727532"}]},"ts":"9223372036854775807"} 2024-12-09T23:58:47,534 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T23:58:47,534 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a41ba930d231212908b11d93742f8c4a, NAME => 'TestAcidGuarantees,,1733788693046.a41ba930d231212908b11d93742f8c4a.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T23:58:47,534 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T23:58:47,534 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733788727534"}]},"ts":"9223372036854775807"} 2024-12-09T23:58:47,536 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T23:58:47,536 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=111, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T23:58:47,537 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 60 msec 2024-12-09T23:58:47,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-09T23:58:47,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T23:58:47,584 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:58:47,599 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=255 (was 258), OpenFileDescriptor=443 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=309 (was 362), ProcessCount=11 (was 11), AvailableMemoryMB=3976 (was 4007) 2024-12-09T23:58:47,608 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=255, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=309, ProcessCount=11, AvailableMemoryMB=3975 2024-12-09T23:58:47,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T23:58:47,609 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T23:58:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T23:58:47,610 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T23:58:47,611 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:58:47,611 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 112 2024-12-09T23:58:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-09T23:58:47,611 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T23:58:47,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742216_1392 (size=1061) 2024-12-09T23:58:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-09T23:58:47,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-09T23:58:48,022 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e 2024-12-09T23:58:48,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742217_1393 (size=53) 2024-12-09T23:58:48,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-09T23:58:48,432 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:58:48,433 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 89b6654328f161896b9d470a5e0e55fb, disabling compactions & flushes 2024-12-09T23:58:48,433 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:48,433 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:48,433 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. after waiting 0 ms 2024-12-09T23:58:48,433 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:48,433 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:48,433 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 89b6654328f161896b9d470a5e0e55fb: Waiting for close lock at 1733788728433Disabling compacts and flushes for region at 1733788728433Disabling writes for close at 1733788728433Writing region close event to WAL at 1733788728433Closed at 1733788728433 2024-12-09T23:58:48,436 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T23:58:48,437 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733788728436"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733788728436"}]},"ts":"1733788728436"} 2024-12-09T23:58:48,440 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T23:58:48,441 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T23:58:48,441 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733788728441"}]},"ts":"1733788728441"} 2024-12-09T23:58:48,442 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T23:58:48,443 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=89b6654328f161896b9d470a5e0e55fb, ASSIGN}] 2024-12-09T23:58:48,444 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=89b6654328f161896b9d470a5e0e55fb, ASSIGN 2024-12-09T23:58:48,445 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=89b6654328f161896b9d470a5e0e55fb, ASSIGN; state=OFFLINE, location=2907d75fbb3e,37627,1733788599837; forceNewPlan=false, retain=false 2024-12-09T23:58:48,596 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=89b6654328f161896b9d470a5e0e55fb, regionState=OPENING, regionLocation=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:48,600 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=89b6654328f161896b9d470a5e0e55fb, ASSIGN because future has completed 2024-12-09T23:58:48,600 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837}] 2024-12-09T23:58:48,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-09T23:58:48,762 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:48,763 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7752): Opening region: {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} 2024-12-09T23:58:48,765 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 89b6654328f161896b9d470a5e0e55fb 2024-12-09T23:58:48,765 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:58:48,765 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7794): checking encryption for 89b6654328f161896b9d470a5e0e55fb 2024-12-09T23:58:48,765 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7797): checking classloading for 89b6654328f161896b9d470a5e0e55fb 2024-12-09T23:58:48,767 INFO [StoreOpener-89b6654328f161896b9d470a5e0e55fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 89b6654328f161896b9d470a5e0e55fb 2024-12-09T23:58:48,768 INFO [StoreOpener-89b6654328f161896b9d470a5e0e55fb-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T23:58:48,769 INFO [StoreOpener-89b6654328f161896b9d470a5e0e55fb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89b6654328f161896b9d470a5e0e55fb columnFamilyName A 2024-12-09T23:58:48,769 DEBUG [StoreOpener-89b6654328f161896b9d470a5e0e55fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:58:48,769 INFO [StoreOpener-89b6654328f161896b9d470a5e0e55fb-1 {}] regionserver.HStore(327): Store=89b6654328f161896b9d470a5e0e55fb/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:58:48,769 INFO [StoreOpener-89b6654328f161896b9d470a5e0e55fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 89b6654328f161896b9d470a5e0e55fb 2024-12-09T23:58:48,771 INFO [StoreOpener-89b6654328f161896b9d470a5e0e55fb-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T23:58:48,771 INFO [StoreOpener-89b6654328f161896b9d470a5e0e55fb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89b6654328f161896b9d470a5e0e55fb columnFamilyName B 2024-12-09T23:58:48,771 DEBUG [StoreOpener-89b6654328f161896b9d470a5e0e55fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:58:48,772 INFO [StoreOpener-89b6654328f161896b9d470a5e0e55fb-1 {}] regionserver.HStore(327): Store=89b6654328f161896b9d470a5e0e55fb/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:58:48,772 INFO [StoreOpener-89b6654328f161896b9d470a5e0e55fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 89b6654328f161896b9d470a5e0e55fb 2024-12-09T23:58:48,773 INFO [StoreOpener-89b6654328f161896b9d470a5e0e55fb-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T23:58:48,773 INFO [StoreOpener-89b6654328f161896b9d470a5e0e55fb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89b6654328f161896b9d470a5e0e55fb columnFamilyName C 2024-12-09T23:58:48,773 DEBUG [StoreOpener-89b6654328f161896b9d470a5e0e55fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:58:48,774 INFO [StoreOpener-89b6654328f161896b9d470a5e0e55fb-1 {}] regionserver.HStore(327): Store=89b6654328f161896b9d470a5e0e55fb/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:58:48,774 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:48,774 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1038): replaying wal for 89b6654328f161896b9d470a5e0e55fb 2024-12-09T23:58:48,775 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb 2024-12-09T23:58:48,775 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb 2024-12-09T23:58:48,777 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1048): stopping wal replay for 89b6654328f161896b9d470a5e0e55fb 2024-12-09T23:58:48,777 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1060): Cleaning up temporary data for 89b6654328f161896b9d470a5e0e55fb 2024-12-09T23:58:48,778 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T23:58:48,780 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1093): writing seq id for 89b6654328f161896b9d470a5e0e55fb 2024-12-09T23:58:48,783 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T23:58:48,784 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1114): Opened 89b6654328f161896b9d470a5e0e55fb; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59830600, jitterRate=-0.10845458507537842}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T23:58:48,784 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 89b6654328f161896b9d470a5e0e55fb 2024-12-09T23:58:48,785 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1006): Region open journal for 89b6654328f161896b9d470a5e0e55fb: Running coprocessor pre-open hook at 1733788728765Writing region info on filesystem at 1733788728765Initializing all the Stores at 1733788728766 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788728766Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788728767 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788728767Cleaning up temporary data from old regions at 1733788728777 (+10 ms)Running coprocessor post-open hooks at 1733788728784 (+7 ms)Region opened successfully at 1733788728785 (+1 ms) 2024-12-09T23:58:48,786 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., pid=114, masterSystemTime=1733788728755 2024-12-09T23:58:48,788 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:48,788 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:48,788 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=89b6654328f161896b9d470a5e0e55fb, regionState=OPEN, openSeqNum=2, regionLocation=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:48,790 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 because future has completed 2024-12-09T23:58:48,792 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-09T23:58:48,792 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; OpenRegionProcedure 89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 in 191 msec 2024-12-09T23:58:48,794 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-12-09T23:58:48,794 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=89b6654328f161896b9d470a5e0e55fb, ASSIGN in 350 msec 2024-12-09T23:58:48,795 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T23:58:48,796 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733788728796"}]},"ts":"1733788728796"} 2024-12-09T23:58:48,797 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T23:58:48,798 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T23:58:48,800 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1890 sec 2024-12-09T23:58:49,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-09T23:58:49,773 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:58:49,774 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6516064e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:49,775 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:58:49,775 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:58:49,776 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:58:49,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:58:49,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:58:49,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ee1c162, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:49,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:58:49,778 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:58:49,778 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:49,780 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54206, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:58:49,781 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f40ded5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:49,781 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:58:49,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:58:49,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:58:49,785 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51350, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:58:49,786 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T23:58:49,787 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 2907d75fbb3e,45191,1733788599119 2024-12-09T23:58:49,787 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1eefce6f 2024-12-09T23:58:49,787 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T23:58:49,788 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54220, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T23:58:49,789 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59de117f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:49,789 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:58:49,789 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:58:49,790 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:58:49,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:58:49,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:58:49,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53d0538e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:49,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:58:49,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:58:49,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:49,791 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:58:49,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27d4f4d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:49,792 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@472a8168, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:49,792 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:58:49,793 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:58:49,793 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:58:49,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:58:49,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:58:49,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bfa775c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:49,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:58:49,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:58:49,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:49,794 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54238, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:58:49,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@459c7238, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:49,796 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f010827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:49,796 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:58:49,796 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:58:49,796 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:58:49,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:58:49,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:58:49,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3741f1e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:49,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:58:49,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:58:49,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:49,798 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54260, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:58:49,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1351d89d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:49,799 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2822c33e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:49,799 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:58:49,800 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:58:49,800 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:58:49,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:58:49,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:58:49,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f8dce99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:49,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:58:49,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:58:49,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:49,802 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54278, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:58:49,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@745788a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:49,804 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f59c6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:49,804 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:58:49,804 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:58:49,805 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:58:49,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:58:49,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:58:49,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8eea82e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:49,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:58:49,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:58:49,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:49,806 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:58:49,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31e4acd8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:49,807 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42daa6ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:49,808 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:58:49,808 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:58:49,808 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:58:49,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:58:49,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:58:49,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5edaa8aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:49,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:58:49,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:58:49,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:49,810 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54306, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:58:49,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@629e4d05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:49,811 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366fb241, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:49,811 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:58:49,812 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:58:49,812 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:58:49,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:58:49,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:58:49,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6895be8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:49,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:58:49,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:58:49,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:49,814 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54326, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:58:49,815 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@676fd583, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:49,816 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45e8e518, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:49,816 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:58:49,816 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:58:49,817 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:58:49,817 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:58:49,817 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:58:49,817 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@634f021f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:49,817 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:58:49,817 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:58:49,818 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:49,818 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54344, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:58:49,819 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@271f574f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:49,820 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10ff7be5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:49,820 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:58:49,820 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:58:49,821 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:58:49,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:58:49,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:58:49,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fff7a1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:49,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:58:49,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:58:49,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:49,822 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54354, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:58:49,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@491a8fe9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:49,824 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55c1b90e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:49,824 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:58:49,824 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:58:49,825 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:58:49,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:58:49,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:58:49,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f67ca92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:49,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:58:49,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:58:49,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:58:49,826 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54374, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:58:49,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a95457e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:58:49,827 DEBUG [Thread-1787 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:58:49,827 DEBUG [Thread-1795 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:58:49,827 DEBUG [Thread-1789 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:58:49,827 DEBUG [Thread-1804 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:58:49,827 DEBUG [Thread-1802 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:58:49,828 DEBUG [Thread-1793 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:58:49,828 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:58:49,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:58:49,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:58:49,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:58:49,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:58:49,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:58:49,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:58:49,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:58:49,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:58:49,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:58:49,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:58:49,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:58:49,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:58:49,829 DEBUG [Thread-1800 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:58:49,829 DEBUG [Thread-1798 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:58:49,829 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51354, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:58:49,829 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51356, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:58:49,830 DEBUG [Thread-1791 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:58:49,830 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51382, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:58:49,830 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51392, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:58:49,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-12-09T23:58:49,830 DEBUG [Thread-1806 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:58:49,830 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51358, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:58:49,830 DEBUG [Thread-1798 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:58:49,830 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:58:49,830 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:58:49,830 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:58:49,830 DEBUG [Thread-1798 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:58:49,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-09T23:58:49,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:58:49,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:58:49,831 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:58:49,832 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51416, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:58:49,832 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:58:49,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:58:49,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2] 2024-12-09T23:58:49,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:58:49,832 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51426, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:58:49,832 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:58:49,832 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:58:49,833 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51430, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:58:49,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2] 2024-12-09T23:58:49,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2] 2024-12-09T23:58:49,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2] 2024-12-09T23:58:49,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2] 2024-12-09T23:58:49,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2] 2024-12-09T23:58:49,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2] 2024-12-09T23:58:49,835 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2] 2024-12-09T23:58:49,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2] 2024-12-09T23:58:49,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2] 2024-12-09T23:58:49,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on 89b6654328f161896b9d470a5e0e55fb 2024-12-09T23:58:49,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 89b6654328f161896b9d470a5e0e55fb 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T23:58:49,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=A 2024-12-09T23:58:49,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:49,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=B 2024-12-09T23:58:49,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:49,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=C 2024-12-09T23:58:49,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:49,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/c7e729ff16924b8990a7ca066e7b2baf is 50, key is test_row_0/A:col10/1733788729841/Put/seqid=0 2024-12-09T23:58:49,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742218_1394 (size=16681) 2024-12-09T23:58:49,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:49,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788789868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:49,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:49,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788789869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:49,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:49,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:49,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:49,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:49,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:49,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:49,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733788729860, max timestamp=1733788729874Num uniques 859; 2024-12-09T23:58:49,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T23:58:49,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733788729860, max timestamp=1733788729874Num uniques 859; 2024-12-09T23:58:49,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733788729860, max timestamp=1733788729874Num uniques 859; 2024-12-09T23:58:49,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T23:58:49,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T23:58:49,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-09T23:58:49,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:49,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788789952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:49,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:49,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:49,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:49,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:49,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788789953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:49,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:49,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:49,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:49,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:49,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51356 deadline: 1733788789954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:49,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:49,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:49,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:49,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-09T23:58:49,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:49,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. as already flushing 2024-12-09T23:58:49,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:49,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:49,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:49,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:50,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:50,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788790002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:50,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:50,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:50,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788790002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:50,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:50,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:50,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:50,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:50,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:50,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:50,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788790063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:50,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:50,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:50,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788790063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:50,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:50,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:50,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:50,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:50,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:50,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:50,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51356 deadline: 1733788790073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:50,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:50,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:50,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:50,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-09T23:58:50,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:50,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. as already flushing 2024-12-09T23:58:50,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:50,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:50,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:50,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:50,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-09T23:58:50,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:50,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788790212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:50,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:50,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:50,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:50,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:50,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788790212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:50,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:50,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:50,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:50,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/c7e729ff16924b8990a7ca066e7b2baf 2024-12-09T23:58:50,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:50,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788790273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:50,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:50,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:50,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:50,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:50,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788790273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:50,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:50,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:50,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:50,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:50,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51356 deadline: 1733788790282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:50,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:50,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:50,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:50,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/c896c0b6a64f4d2db280060e423978b9 is 50, key is test_row_0/B:col10/1733788729841/Put/seqid=0 2024-12-09T23:58:50,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-09T23:58:50,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:50,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. as already flushing 2024-12-09T23:58:50,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:50,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:50,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742219_1395 (size=12001) 2024-12-09T23:58:50,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:50,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:50,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/c896c0b6a64f4d2db280060e423978b9 2024-12-09T23:58:50,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/72349c755c0c497095dd54a8bf095ab5 is 50, key is test_row_0/C:col10/1733788729841/Put/seqid=0 2024-12-09T23:58:50,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742220_1396 (size=12001) 2024-12-09T23:58:50,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-09T23:58:50,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:50,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. as already flushing 2024-12-09T23:58:50,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:50,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:50,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:50,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:50,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-09T23:58:50,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:50,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788790523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:50,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:50,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:50,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:50,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:50,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788790532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:50,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:50,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:50,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:50,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:50,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788790583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:50,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:50,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:50,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:50,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:50,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788790583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:50,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:50,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:50,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:50,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-09T23:58:50,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:50,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:50,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. as already flushing 2024-12-09T23:58:50,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51356 deadline: 1733788790593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:50,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:50,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:50,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:50,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:50,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:50,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:50,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:50,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/72349c755c0c497095dd54a8bf095ab5 2024-12-09T23:58:50,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/c7e729ff16924b8990a7ca066e7b2baf as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/c7e729ff16924b8990a7ca066e7b2baf 2024-12-09T23:58:50,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/c7e729ff16924b8990a7ca066e7b2baf, entries=250, sequenceid=12, filesize=16.3 K 2024-12-09T23:58:50,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/c896c0b6a64f4d2db280060e423978b9 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/c896c0b6a64f4d2db280060e423978b9 2024-12-09T23:58:50,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/c896c0b6a64f4d2db280060e423978b9, entries=150, sequenceid=12, filesize=11.7 K 2024-12-09T23:58:50,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/72349c755c0c497095dd54a8bf095ab5 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/72349c755c0c497095dd54a8bf095ab5 2024-12-09T23:58:50,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/72349c755c0c497095dd54a8bf095ab5, entries=150, sequenceid=12, filesize=11.7 K 2024-12-09T23:58:50,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 89b6654328f161896b9d470a5e0e55fb in 890ms, sequenceid=12, compaction requested=false 2024-12-09T23:58:50,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:58:50,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-09T23:58:50,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:50,751 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 89b6654328f161896b9d470a5e0e55fb 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-09T23:58:50,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=A 2024-12-09T23:58:50,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:58:50,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=B 2024-12-09T23:58:50,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:58:50,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=C 2024-12-09T23:58:50,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:58:50,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/6ef155a2634140d3b3bdbd9c55d5caea is 50, key is test_row_0/A:col10/1733788729952/Put/seqid=0 2024-12-09T23:58:50,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742221_1397 (size=12001) 2024-12-09T23:58:50,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-09T23:58:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on 89b6654328f161896b9d470a5e0e55fb 2024-12-09T23:58:51,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. as already flushing 2024-12-09T23:58:51,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:51,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788791056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:51,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:51,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:51,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:51,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788791057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:51,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:51,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:51,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:51,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:51,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788791092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:51,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:51,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:51,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:51,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:51,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788791092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:51,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:51,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:51,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:51,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:51,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51356 deadline: 1733788791102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:51,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:51,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:51,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:51,158 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/6ef155a2634140d3b3bdbd9c55d5caea 2024-12-09T23:58:51,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/bd09b9f94c89468a86af9dd394031de3 is 50, key is test_row_0/B:col10/1733788729952/Put/seqid=0 2024-12-09T23:58:51,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:51,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788791163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:51,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:51,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:51,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:51,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742222_1398 (size=12001) 2024-12-09T23:58:51,168 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/bd09b9f94c89468a86af9dd394031de3 2024-12-09T23:58:51,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:51,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788791172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:51,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/5d9a2ff3e62348fb8a854d62932d63bb is 50, key is test_row_0/C:col10/1733788729952/Put/seqid=0 2024-12-09T23:58:51,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:51,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:51,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:51,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742223_1399 (size=12001) 2024-12-09T23:58:51,183 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/5d9a2ff3e62348fb8a854d62932d63bb 2024-12-09T23:58:51,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/6ef155a2634140d3b3bdbd9c55d5caea as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/6ef155a2634140d3b3bdbd9c55d5caea 2024-12-09T23:58:51,190 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/6ef155a2634140d3b3bdbd9c55d5caea, entries=150, sequenceid=46, filesize=11.7 K 2024-12-09T23:58:51,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/bd09b9f94c89468a86af9dd394031de3 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/bd09b9f94c89468a86af9dd394031de3 2024-12-09T23:58:51,194 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/bd09b9f94c89468a86af9dd394031de3, entries=150, sequenceid=46, filesize=11.7 K 2024-12-09T23:58:51,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/5d9a2ff3e62348fb8a854d62932d63bb as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/5d9a2ff3e62348fb8a854d62932d63bb 2024-12-09T23:58:51,198 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/5d9a2ff3e62348fb8a854d62932d63bb, entries=150, sequenceid=46, filesize=11.7 K 2024-12-09T23:58:51,199 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 89b6654328f161896b9d470a5e0e55fb in 449ms, sequenceid=46, compaction requested=false 2024-12-09T23:58:51,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:58:51,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:51,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-12-09T23:58:51,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-12-09T23:58:51,202 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-09T23:58:51,202 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3680 sec 2024-12-09T23:58:51,204 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.3750 sec 2024-12-09T23:58:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on 89b6654328f161896b9d470a5e0e55fb 2024-12-09T23:58:51,375 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 89b6654328f161896b9d470a5e0e55fb 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T23:58:51,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=A 2024-12-09T23:58:51,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:51,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=B 2024-12-09T23:58:51,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:51,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=C 2024-12-09T23:58:51,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:51,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/c97c63a9176a4f8cbad55f3bae789f5f is 50, key is test_row_0/A:col10/1733788731056/Put/seqid=0 2024-12-09T23:58:51,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742224_1400 (size=11997) 2024-12-09T23:58:51,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788791421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:51,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788791423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:51,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:51,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:51,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:51,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:51,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:51,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:51,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788791532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:51,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788791532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:51,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:51,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:51,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:51,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:51,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:51,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:51,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:51,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788791743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:51,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:51,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:51,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788791743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:51,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:51,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:51,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:51,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:51,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:51,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/c97c63a9176a4f8cbad55f3bae789f5f 2024-12-09T23:58:51,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/78b13749d4614df1876158633c245de1 is 50, key is test_row_0/B:col10/1733788731056/Put/seqid=0 2024-12-09T23:58:51,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742225_1401 (size=9657) 2024-12-09T23:58:51,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/78b13749d4614df1876158633c245de1 2024-12-09T23:58:51,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/3d49a9ebbcf24c128b41bb57e881a93b is 50, key is test_row_0/C:col10/1733788731056/Put/seqid=0 2024-12-09T23:58:51,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742226_1402 (size=9657) 2024-12-09T23:58:51,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-09T23:58:51,983 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:58:51,983 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:58:51,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-12-09T23:58:51,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-09T23:58:51,985 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:58:51,986 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:58:51,986 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:58:52,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:52,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788792052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:52,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:52,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:52,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:52,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:52,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788792053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:52,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:52,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:52,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:52,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-09T23:58:52,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:52,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788792112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:52,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:52,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788792112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:52,116 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_2' in 89b6654328f161896b9d470a5e0e55fb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:58:52,116 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_1' in 89b6654328f161896b9d470a5e0e55fb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:58:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:52,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:52,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51356 deadline: 1733788792123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:52,128 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_2' in 89b6654328f161896b9d470a5e0e55fb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:58:52,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:52,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:52,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:52,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-09T23:58:52,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:52,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. as already flushing 2024-12-09T23:58:52,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:52,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:52,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:52,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:52,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/3d49a9ebbcf24c128b41bb57e881a93b 2024-12-09T23:58:52,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/c97c63a9176a4f8cbad55f3bae789f5f as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/c97c63a9176a4f8cbad55f3bae789f5f 2024-12-09T23:58:52,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/c97c63a9176a4f8cbad55f3bae789f5f, entries=150, sequenceid=58, filesize=11.7 K 2024-12-09T23:58:52,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/78b13749d4614df1876158633c245de1 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/78b13749d4614df1876158633c245de1 2024-12-09T23:58:52,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/78b13749d4614df1876158633c245de1, entries=100, sequenceid=58, filesize=9.4 K 2024-12-09T23:58:52,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/3d49a9ebbcf24c128b41bb57e881a93b as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/3d49a9ebbcf24c128b41bb57e881a93b 2024-12-09T23:58:52,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/3d49a9ebbcf24c128b41bb57e881a93b, entries=100, sequenceid=58, filesize=9.4 K 2024-12-09T23:58:52,262 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 89b6654328f161896b9d470a5e0e55fb in 887ms, sequenceid=58, compaction requested=true 2024-12-09T23:58:52,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:58:52,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89b6654328f161896b9d470a5e0e55fb:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:58:52,262 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:58:52,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:52,262 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:58:52,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89b6654328f161896b9d470a5e0e55fb:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:58:52,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:52,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89b6654328f161896b9d470a5e0e55fb:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:58:52,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:58:52,263 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40679 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:58:52,263 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:58:52,263 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): 89b6654328f161896b9d470a5e0e55fb/B is initiating minor compaction (all files) 2024-12-09T23:58:52,263 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): 89b6654328f161896b9d470a5e0e55fb/A is initiating minor compaction (all files) 2024-12-09T23:58:52,263 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89b6654328f161896b9d470a5e0e55fb/B in TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:52,263 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89b6654328f161896b9d470a5e0e55fb/A in TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:52,263 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/c7e729ff16924b8990a7ca066e7b2baf, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/6ef155a2634140d3b3bdbd9c55d5caea, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/c97c63a9176a4f8cbad55f3bae789f5f] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp, totalSize=39.7 K 2024-12-09T23:58:52,263 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/c896c0b6a64f4d2db280060e423978b9, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/bd09b9f94c89468a86af9dd394031de3, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/78b13749d4614df1876158633c245de1] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp, totalSize=32.9 K 2024-12-09T23:58:52,263 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting c896c0b6a64f4d2db280060e423978b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733788729839 2024-12-09T23:58:52,263 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7e729ff16924b8990a7ca066e7b2baf, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733788729837 2024-12-09T23:58:52,264 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting bd09b9f94c89468a86af9dd394031de3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733788729917 2024-12-09T23:58:52,264 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ef155a2634140d3b3bdbd9c55d5caea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733788729917 2024-12-09T23:58:52,264 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 78b13749d4614df1876158633c245de1, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733788731056 2024-12-09T23:58:52,264 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting c97c63a9176a4f8cbad55f3bae789f5f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733788731055 2024-12-09T23:58:52,270 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89b6654328f161896b9d470a5e0e55fb#B#compaction#336 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:52,270 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/02a8981061ed4334bb99c04a2af8a326 is 50, key is test_row_0/B:col10/1733788731056/Put/seqid=0 2024-12-09T23:58:52,272 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89b6654328f161896b9d470a5e0e55fb#A#compaction#337 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:52,272 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/fd7a6afe120341c9800a5f9a7eadc55f is 50, key is test_row_0/A:col10/1733788731056/Put/seqid=0 2024-12-09T23:58:52,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742227_1403 (size=12104) 2024-12-09T23:58:52,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742228_1404 (size=12104) 2024-12-09T23:58:52,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-09T23:58:52,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:52,289 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 89b6654328f161896b9d470a5e0e55fb 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T23:58:52,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=A 2024-12-09T23:58:52,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:52,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=B 2024-12-09T23:58:52,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:52,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=C 2024-12-09T23:58:52,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:52,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/b0d0d55c26f147b697d2f16bcfbc03ea is 50, key is test_row_0/A:col10/1733788731419/Put/seqid=0 2024-12-09T23:58:52,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742229_1405 (size=12001) 2024-12-09T23:58:52,298 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/b0d0d55c26f147b697d2f16bcfbc03ea 2024-12-09T23:58:52,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-09T23:58:52,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/46e4fca786fd4fad8211f25602e3835b is 50, key is test_row_0/B:col10/1733788731419/Put/seqid=0 2024-12-09T23:58:52,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742230_1406 (size=12001) 2024-12-09T23:58:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on 89b6654328f161896b9d470a5e0e55fb 2024-12-09T23:58:52,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. as already flushing 2024-12-09T23:58:52,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:52,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788792589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:52,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:52,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:52,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:52,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:52,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788792595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:52,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:52,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:52,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:52,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-09T23:58:52,685 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/02a8981061ed4334bb99c04a2af8a326 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/02a8981061ed4334bb99c04a2af8a326 2024-12-09T23:58:52,690 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89b6654328f161896b9d470a5e0e55fb/B of 89b6654328f161896b9d470a5e0e55fb into 02a8981061ed4334bb99c04a2af8a326(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:52,690 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:58:52,690 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., storeName=89b6654328f161896b9d470a5e0e55fb/B, priority=13, startTime=1733788732262; duration=0sec 2024-12-09T23:58:52,690 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:58:52,690 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89b6654328f161896b9d470a5e0e55fb:B 2024-12-09T23:58:52,690 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:58:52,690 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/fd7a6afe120341c9800a5f9a7eadc55f as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/fd7a6afe120341c9800a5f9a7eadc55f 2024-12-09T23:58:52,691 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:58:52,691 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): 89b6654328f161896b9d470a5e0e55fb/C is initiating minor compaction (all files) 2024-12-09T23:58:52,691 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89b6654328f161896b9d470a5e0e55fb/C in TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:52,691 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/72349c755c0c497095dd54a8bf095ab5, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/5d9a2ff3e62348fb8a854d62932d63bb, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/3d49a9ebbcf24c128b41bb57e881a93b] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp, totalSize=32.9 K 2024-12-09T23:58:52,691 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 72349c755c0c497095dd54a8bf095ab5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733788729839 2024-12-09T23:58:52,692 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d9a2ff3e62348fb8a854d62932d63bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733788729917 2024-12-09T23:58:52,692 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d49a9ebbcf24c128b41bb57e881a93b, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733788731056 2024-12-09T23:58:52,695 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89b6654328f161896b9d470a5e0e55fb/A of 89b6654328f161896b9d470a5e0e55fb into fd7a6afe120341c9800a5f9a7eadc55f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:52,695 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:58:52,695 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., storeName=89b6654328f161896b9d470a5e0e55fb/A, priority=13, startTime=1733788732262; duration=0sec 2024-12-09T23:58:52,695 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:52,695 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89b6654328f161896b9d470a5e0e55fb:A 2024-12-09T23:58:52,699 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89b6654328f161896b9d470a5e0e55fb#C#compaction#340 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:52,699 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/01a2f54f7d784faa8cf5505191b5ae27 is 50, key is test_row_0/C:col10/1733788731056/Put/seqid=0 2024-12-09T23:58:52,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742231_1407 (size=12104) 2024-12-09T23:58:52,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:52,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788792702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:52,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:52,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788792703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:52,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:52,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:52,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:52,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:52,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:52,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:52,709 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/46e4fca786fd4fad8211f25602e3835b 2024-12-09T23:58:52,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/fa2d72bfc22b4558baba577797bfba71 is 50, key is test_row_0/C:col10/1733788731419/Put/seqid=0 2024-12-09T23:58:52,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742232_1408 (size=12001) 2024-12-09T23:58:52,735 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T23:58:52,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:52,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788792913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:52,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:52,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:52,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:52,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:52,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788792913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:52,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:52,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:52,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:53,107 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/01a2f54f7d784faa8cf5505191b5ae27 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/01a2f54f7d784faa8cf5505191b5ae27 2024-12-09T23:58:53,111 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89b6654328f161896b9d470a5e0e55fb/C of 89b6654328f161896b9d470a5e0e55fb into 01a2f54f7d784faa8cf5505191b5ae27(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:53,111 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:58:53,111 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., storeName=89b6654328f161896b9d470a5e0e55fb/C, priority=13, startTime=1733788732262; duration=0sec 2024-12-09T23:58:53,111 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:53,111 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89b6654328f161896b9d470a5e0e55fb:C 2024-12-09T23:58:53,118 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/fa2d72bfc22b4558baba577797bfba71 2024-12-09T23:58:53,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/b0d0d55c26f147b697d2f16bcfbc03ea as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/b0d0d55c26f147b697d2f16bcfbc03ea 2024-12-09T23:58:53,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-09T23:58:53,124 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/b0d0d55c26f147b697d2f16bcfbc03ea, entries=150, sequenceid=78, filesize=11.7 K 2024-12-09T23:58:53,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/46e4fca786fd4fad8211f25602e3835b as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/46e4fca786fd4fad8211f25602e3835b 2024-12-09T23:58:53,127 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/46e4fca786fd4fad8211f25602e3835b, entries=150, sequenceid=78, filesize=11.7 K 2024-12-09T23:58:53,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/fa2d72bfc22b4558baba577797bfba71 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/fa2d72bfc22b4558baba577797bfba71 2024-12-09T23:58:53,131 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/fa2d72bfc22b4558baba577797bfba71, entries=150, sequenceid=78, filesize=11.7 K 2024-12-09T23:58:53,131 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 89b6654328f161896b9d470a5e0e55fb in 842ms, sequenceid=78, compaction requested=false 2024-12-09T23:58:53,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:58:53,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:53,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-12-09T23:58:53,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-12-09T23:58:53,134 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-09T23:58:53,134 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1470 sec 2024-12-09T23:58:53,135 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.1500 sec 2024-12-09T23:58:53,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on 89b6654328f161896b9d470a5e0e55fb 2024-12-09T23:58:53,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 89b6654328f161896b9d470a5e0e55fb 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T23:58:53,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=A 2024-12-09T23:58:53,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:53,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=B 2024-12-09T23:58:53,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:53,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=C 2024-12-09T23:58:53,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:53,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/c5f24ed5ffe1458d975964891cc2ec63 is 50, key is test_row_0/A:col10/1733788732586/Put/seqid=0 2024-12-09T23:58:53,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742233_1409 (size=16681) 2024-12-09T23:58:53,245 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/c5f24ed5ffe1458d975964891cc2ec63 2024-12-09T23:58:53,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/b1f2b5c8738c4006ad9a459a0e5091d8 is 50, key is test_row_0/B:col10/1733788732586/Put/seqid=0 2024-12-09T23:58:53,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742234_1410 (size=12001) 2024-12-09T23:58:53,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:53,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788793285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:53,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:53,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:53,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:53,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:53,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788793285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:53,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:53,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:53,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:53,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:53,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788793393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:53,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:53,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:53,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:53,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:53,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788793402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:53,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:53,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:53,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:53,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:53,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788793603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:53,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:53,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:53,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:53,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:53,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788793613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:53,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:53,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:53,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:53,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/b1f2b5c8738c4006ad9a459a0e5091d8 2024-12-09T23:58:53,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/7941b97b10b941e3ad29b4681bbcdd1a is 50, key is test_row_0/C:col10/1733788732586/Put/seqid=0 2024-12-09T23:58:53,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742235_1411 (size=12001) 2024-12-09T23:58:53,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:53,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788793913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:53,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:53,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:53,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:53,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:53,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788793922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:53,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:53,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:53,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:54,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/7941b97b10b941e3ad29b4681bbcdd1a 2024-12-09T23:58:54,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/c5f24ed5ffe1458d975964891cc2ec63 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/c5f24ed5ffe1458d975964891cc2ec63 2024-12-09T23:58:54,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/c5f24ed5ffe1458d975964891cc2ec63, entries=250, sequenceid=95, filesize=16.3 K 2024-12-09T23:58:54,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/b1f2b5c8738c4006ad9a459a0e5091d8 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/b1f2b5c8738c4006ad9a459a0e5091d8 2024-12-09T23:58:54,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/b1f2b5c8738c4006ad9a459a0e5091d8, entries=150, sequenceid=95, filesize=11.7 K 2024-12-09T23:58:54,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/7941b97b10b941e3ad29b4681bbcdd1a as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/7941b97b10b941e3ad29b4681bbcdd1a 2024-12-09T23:58:54,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/7941b97b10b941e3ad29b4681bbcdd1a, entries=150, sequenceid=95, filesize=11.7 K 2024-12-09T23:58:54,077 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 89b6654328f161896b9d470a5e0e55fb in 840ms, sequenceid=95, compaction requested=true 2024-12-09T23:58:54,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:58:54,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89b6654328f161896b9d470a5e0e55fb:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:58:54,077 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:58:54,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:54,077 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:58:54,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89b6654328f161896b9d470a5e0e55fb:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:58:54,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:54,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89b6654328f161896b9d470a5e0e55fb:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:58:54,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:58:54,078 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:58:54,078 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:58:54,078 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): 89b6654328f161896b9d470a5e0e55fb/B is initiating minor compaction (all files) 2024-12-09T23:58:54,078 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): 89b6654328f161896b9d470a5e0e55fb/A is initiating minor compaction (all files) 2024-12-09T23:58:54,078 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89b6654328f161896b9d470a5e0e55fb/B in TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:54,078 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89b6654328f161896b9d470a5e0e55fb/A in TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:54,078 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/02a8981061ed4334bb99c04a2af8a326, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/46e4fca786fd4fad8211f25602e3835b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/b1f2b5c8738c4006ad9a459a0e5091d8] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp, totalSize=35.3 K 2024-12-09T23:58:54,078 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/fd7a6afe120341c9800a5f9a7eadc55f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/b0d0d55c26f147b697d2f16bcfbc03ea, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/c5f24ed5ffe1458d975964891cc2ec63] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp, totalSize=39.8 K 2024-12-09T23:58:54,078 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 02a8981061ed4334bb99c04a2af8a326, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733788729917 2024-12-09T23:58:54,078 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd7a6afe120341c9800a5f9a7eadc55f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733788729917 2024-12-09T23:58:54,078 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 46e4fca786fd4fad8211f25602e3835b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733788731414 2024-12-09T23:58:54,078 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0d0d55c26f147b697d2f16bcfbc03ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733788731414 2024-12-09T23:58:54,078 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting b1f2b5c8738c4006ad9a459a0e5091d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733788732586 2024-12-09T23:58:54,079 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5f24ed5ffe1458d975964891cc2ec63, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733788732586 2024-12-09T23:58:54,083 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89b6654328f161896b9d470a5e0e55fb#A#compaction#345 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:54,084 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/c14e98b884d74b20aca964a1b1cd07e7 is 50, key is test_row_0/A:col10/1733788732586/Put/seqid=0 2024-12-09T23:58:54,084 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89b6654328f161896b9d470a5e0e55fb#B#compaction#346 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:54,084 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/f8e7445498e848029d39914c06145d3a is 50, key is test_row_0/B:col10/1733788732586/Put/seqid=0 2024-12-09T23:58:54,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742237_1413 (size=12207) 2024-12-09T23:58:54,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742236_1412 (size=12207) 2024-12-09T23:58:54,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-09T23:58:54,133 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:58:54,134 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:58:54,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-12-09T23:58:54,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-09T23:58:54,135 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:58:54,136 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:58:54,136 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:58:54,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on 89b6654328f161896b9d470a5e0e55fb 2024-12-09T23:58:54,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 89b6654328f161896b9d470a5e0e55fb 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T23:58:54,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=A 2024-12-09T23:58:54,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:54,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=B 2024-12-09T23:58:54,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:54,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=C 2024-12-09T23:58:54,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:54,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/8b5ceea34c55443cb7f00f5c65a8bfec is 50, key is test_row_0/A:col10/1733788733266/Put/seqid=0 2024-12-09T23:58:54,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742238_1414 (size=14341) 2024-12-09T23:58:54,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:54,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788794170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:54,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:54,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:54,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:54,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:54,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51356 deadline: 1733788794171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:54,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:54,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:54,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:54,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:54,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788794180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:54,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-09T23:58:54,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-09T23:58:54,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:54,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. as already flushing 2024-12-09T23:58:54,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:54,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:54,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:54,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788794282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:54,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:54,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:54,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:54,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51356 deadline: 1733788794283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:54,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:54,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:54,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:54,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:54,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788794293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:54,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:54,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:54,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:54,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:54,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788794423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:54,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:54,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:54,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:54,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:54,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788794433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:54,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:54,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:54,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:54,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-09T23:58:54,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:54,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. as already flushing 2024-12-09T23:58:54,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:54,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:54,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:54,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:54,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-09T23:58:54,492 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/f8e7445498e848029d39914c06145d3a as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/f8e7445498e848029d39914c06145d3a 2024-12-09T23:58:54,492 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/c14e98b884d74b20aca964a1b1cd07e7 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/c14e98b884d74b20aca964a1b1cd07e7 2024-12-09T23:58:54,496 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89b6654328f161896b9d470a5e0e55fb/A of 89b6654328f161896b9d470a5e0e55fb into c14e98b884d74b20aca964a1b1cd07e7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:54,496 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89b6654328f161896b9d470a5e0e55fb/B of 89b6654328f161896b9d470a5e0e55fb into f8e7445498e848029d39914c06145d3a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:54,496 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:58:54,496 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:58:54,496 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., storeName=89b6654328f161896b9d470a5e0e55fb/A, priority=13, startTime=1733788734077; duration=0sec 2024-12-09T23:58:54,496 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., storeName=89b6654328f161896b9d470a5e0e55fb/B, priority=13, startTime=1733788734077; duration=0sec 2024-12-09T23:58:54,496 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:58:54,496 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89b6654328f161896b9d470a5e0e55fb:A 2024-12-09T23:58:54,496 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:54,497 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89b6654328f161896b9d470a5e0e55fb:B 2024-12-09T23:58:54,497 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:58:54,497 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:58:54,497 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): 89b6654328f161896b9d470a5e0e55fb/C is initiating minor compaction (all files) 2024-12-09T23:58:54,497 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89b6654328f161896b9d470a5e0e55fb/C in TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:54,497 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/01a2f54f7d784faa8cf5505191b5ae27, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/fa2d72bfc22b4558baba577797bfba71, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/7941b97b10b941e3ad29b4681bbcdd1a] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp, totalSize=35.3 K 2024-12-09T23:58:54,498 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01a2f54f7d784faa8cf5505191b5ae27, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733788729917 2024-12-09T23:58:54,498 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa2d72bfc22b4558baba577797bfba71, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733788731414 2024-12-09T23:58:54,498 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7941b97b10b941e3ad29b4681bbcdd1a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733788732586 2024-12-09T23:58:54,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:54,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788794493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:54,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:54,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51356 deadline: 1733788794493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:54,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:54,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:54,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:54,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:54,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:54,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:54,506 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89b6654328f161896b9d470a5e0e55fb#C#compaction#348 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:54,506 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/9930a9d5f49e4747842b5337a28cda8b is 50, key is test_row_0/C:col10/1733788732586/Put/seqid=0 2024-12-09T23:58:54,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742239_1415 (size=12207) 2024-12-09T23:58:54,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788794503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:54,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:54,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:54,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:54,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/8b5ceea34c55443cb7f00f5c65a8bfec 2024-12-09T23:58:54,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/e2f2225719004f9d9f3e8bf1367a69c9 is 50, key is test_row_0/B:col10/1733788733266/Put/seqid=0 2024-12-09T23:58:54,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742240_1416 (size=12001) 2024-12-09T23:58:54,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-09T23:58:54,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:54,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. as already flushing 2024-12-09T23:58:54,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:54,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:54,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:54,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:54,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-09T23:58:54,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:54,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. as already flushing 2024-12-09T23:58:54,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:54,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:54,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:54,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:54,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-09T23:58:54,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788794813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:54,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:54,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:54,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:54,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51356 deadline: 1733788794813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:54,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:54,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:54,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:54,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788794823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:54,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:54,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:54,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:54,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-09T23:58:54,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:54,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. as already flushing 2024-12-09T23:58:54,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:54,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:54,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:54,913 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/9930a9d5f49e4747842b5337a28cda8b as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/9930a9d5f49e4747842b5337a28cda8b 2024-12-09T23:58:54,917 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89b6654328f161896b9d470a5e0e55fb/C of 89b6654328f161896b9d470a5e0e55fb into 9930a9d5f49e4747842b5337a28cda8b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:54,917 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:58:54,917 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., storeName=89b6654328f161896b9d470a5e0e55fb/C, priority=13, startTime=1733788734077; duration=0sec 2024-12-09T23:58:54,917 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:54,917 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89b6654328f161896b9d470a5e0e55fb:C 2024-12-09T23:58:54,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/e2f2225719004f9d9f3e8bf1367a69c9 2024-12-09T23:58:54,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/fb215f3cb33f4119a25d2f1a911156e8 is 50, key is test_row_0/C:col10/1733788733266/Put/seqid=0 2024-12-09T23:58:54,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742241_1417 (size=12001) 2024-12-09T23:58:54,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/fb215f3cb33f4119a25d2f1a911156e8 2024-12-09T23:58:54,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/8b5ceea34c55443cb7f00f5c65a8bfec as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/8b5ceea34c55443cb7f00f5c65a8bfec 2024-12-09T23:58:54,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/8b5ceea34c55443cb7f00f5c65a8bfec, entries=200, sequenceid=114, filesize=14.0 K 2024-12-09T23:58:54,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/e2f2225719004f9d9f3e8bf1367a69c9 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/e2f2225719004f9d9f3e8bf1367a69c9 2024-12-09T23:58:54,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/e2f2225719004f9d9f3e8bf1367a69c9, entries=150, sequenceid=114, filesize=11.7 K 2024-12-09T23:58:54,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/fb215f3cb33f4119a25d2f1a911156e8 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/fb215f3cb33f4119a25d2f1a911156e8 2024-12-09T23:58:54,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/fb215f3cb33f4119a25d2f1a911156e8, entries=150, sequenceid=114, filesize=11.7 K 2024-12-09T23:58:54,987 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 89b6654328f161896b9d470a5e0e55fb in 848ms, sequenceid=114, compaction requested=false 2024-12-09T23:58:54,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:58:55,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-09T23:58:55,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:55,051 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 89b6654328f161896b9d470a5e0e55fb 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T23:58:55,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=A 2024-12-09T23:58:55,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:55,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=B 2024-12-09T23:58:55,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:55,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=C 2024-12-09T23:58:55,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:55,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/743c280d45aa4d038ad8bdf59e2eec47 is 50, key is test_row_0/A:col10/1733788734179/Put/seqid=0 2024-12-09T23:58:55,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742242_1418 (size=12001) 2024-12-09T23:58:55,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-09T23:58:55,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on 89b6654328f161896b9d470a5e0e55fb 2024-12-09T23:58:55,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. as already flushing 2024-12-09T23:58:55,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:55,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788795355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:55,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:55,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51356 deadline: 1733788795356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:55,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:55,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:55,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:55,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:55,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:55,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:55,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:55,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788795357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:55,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:55,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:55,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:55,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:55,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788795443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:55,444 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_2' in 89b6654328f161896b9d470a5e0e55fb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:58:55,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:55,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:55,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:55,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788795453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:55,456 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_1' in 89b6654328f161896b9d470a5e0e55fb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:58:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:55,458 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/743c280d45aa4d038ad8bdf59e2eec47 2024-12-09T23:58:55,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/e153ac1233d547459831267a8c171520 is 50, key is test_row_0/B:col10/1733788734179/Put/seqid=0 2024-12-09T23:58:55,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742243_1419 (size=12001) 2024-12-09T23:58:55,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:55,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788795472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:55,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:55,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:55,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:55,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:55,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51356 deadline: 1733788795472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:55,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:55,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:55,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:55,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:55,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788795473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:55,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:55,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:55,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:55,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:55,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788795683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:55,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:55,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:55,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:55,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788795683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:55,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:55,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:55,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:55,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51356 deadline: 1733788795683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:55,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:55,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:55,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:55,867 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/e153ac1233d547459831267a8c171520 2024-12-09T23:58:55,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/991021ce8d35487daf105b64c7dab6ef is 50, key is test_row_0/C:col10/1733788734179/Put/seqid=0 2024-12-09T23:58:55,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742244_1420 (size=12001) 2024-12-09T23:58:55,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:55,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788795992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:55,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:55,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:55,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:56,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788796003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:56,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:56,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:56,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:56,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51356 deadline: 1733788796003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:56,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:56,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:56,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:56,275 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/991021ce8d35487daf105b64c7dab6ef 2024-12-09T23:58:56,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/743c280d45aa4d038ad8bdf59e2eec47 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/743c280d45aa4d038ad8bdf59e2eec47 2024-12-09T23:58:56,281 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/743c280d45aa4d038ad8bdf59e2eec47, entries=150, sequenceid=130, filesize=11.7 K 2024-12-09T23:58:56,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/e153ac1233d547459831267a8c171520 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/e153ac1233d547459831267a8c171520 2024-12-09T23:58:56,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-09T23:58:56,284 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/e153ac1233d547459831267a8c171520, entries=150, sequenceid=130, filesize=11.7 K 2024-12-09T23:58:56,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/991021ce8d35487daf105b64c7dab6ef as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/991021ce8d35487daf105b64c7dab6ef 2024-12-09T23:58:56,287 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/991021ce8d35487daf105b64c7dab6ef, entries=150, sequenceid=130, filesize=11.7 K 2024-12-09T23:58:56,288 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 89b6654328f161896b9d470a5e0e55fb in 1237ms, sequenceid=130, compaction requested=true 2024-12-09T23:58:56,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:58:56,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:56,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-09T23:58:56,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-09T23:58:56,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-09T23:58:56,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1530 sec 2024-12-09T23:58:56,292 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.1570 sec 2024-12-09T23:58:56,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on 89b6654328f161896b9d470a5e0e55fb 2024-12-09T23:58:56,506 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 89b6654328f161896b9d470a5e0e55fb 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T23:58:56,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=A 2024-12-09T23:58:56,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:56,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=B 2024-12-09T23:58:56,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:56,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=C 2024-12-09T23:58:56,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:56,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/2c59f1d7c7854531bef87a2130fe21c6 is 50, key is test_row_0/A:col10/1733788735355/Put/seqid=0 2024-12-09T23:58:56,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742245_1421 (size=14541) 2024-12-09T23:58:56,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:56,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788796525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:56,529 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:56,530 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:56,530 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:56,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:56,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788796528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:56,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:56,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:56,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:56,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:56,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51356 deadline: 1733788796529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:56,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:56,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:56,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:56,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:56,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788796633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:56,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:56,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:56,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:56,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:56,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788796643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:56,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:56,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:56,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:56,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:56,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51356 deadline: 1733788796643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:56,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:56,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:56,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:56,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:56,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788796842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:56,845 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:56,845 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:56,845 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:56,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:56,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788796853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:56,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:56,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:56,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:56,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:56,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51356 deadline: 1733788796853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:56,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:56,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:56,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:56,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/2c59f1d7c7854531bef87a2130fe21c6 2024-12-09T23:58:56,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/ba422969416b4f919090eef2270d0e1e is 50, key is test_row_0/B:col10/1733788735355/Put/seqid=0 2024-12-09T23:58:56,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742246_1422 (size=12151) 2024-12-09T23:58:56,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/ba422969416b4f919090eef2270d0e1e 2024-12-09T23:58:56,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/6a6ecfa610b341cfae8f7f0ac156961d is 50, key is test_row_0/C:col10/1733788735355/Put/seqid=0 2024-12-09T23:58:56,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742247_1423 (size=12151) 2024-12-09T23:58:57,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:57,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788797153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:57,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:57,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:57,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:57,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788797162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:57,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:57,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:57,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:57,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51356 deadline: 1733788797162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:57,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:57,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:57,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:57,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/6a6ecfa610b341cfae8f7f0ac156961d 2024-12-09T23:58:57,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/2c59f1d7c7854531bef87a2130fe21c6 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/2c59f1d7c7854531bef87a2130fe21c6 2024-12-09T23:58:57,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/2c59f1d7c7854531bef87a2130fe21c6, entries=200, sequenceid=150, filesize=14.2 K 2024-12-09T23:58:57,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/ba422969416b4f919090eef2270d0e1e as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/ba422969416b4f919090eef2270d0e1e 2024-12-09T23:58:57,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/ba422969416b4f919090eef2270d0e1e, entries=150, sequenceid=150, filesize=11.9 K 2024-12-09T23:58:57,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/6a6ecfa610b341cfae8f7f0ac156961d as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/6a6ecfa610b341cfae8f7f0ac156961d 2024-12-09T23:58:57,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/6a6ecfa610b341cfae8f7f0ac156961d, entries=150, sequenceid=150, filesize=11.9 K 2024-12-09T23:58:57,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 89b6654328f161896b9d470a5e0e55fb in 845ms, sequenceid=150, compaction requested=true 2024-12-09T23:58:57,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:58:57,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89b6654328f161896b9d470a5e0e55fb:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:58:57,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:57,351 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T23:58:57,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89b6654328f161896b9d470a5e0e55fb:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:58:57,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:57,351 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T23:58:57,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89b6654328f161896b9d470a5e0e55fb:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:58:57,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:58:57,352 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T23:58:57,352 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T23:58:57,352 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): 89b6654328f161896b9d470a5e0e55fb/B is initiating minor compaction (all files) 2024-12-09T23:58:57,352 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): 89b6654328f161896b9d470a5e0e55fb/A is initiating minor compaction (all files) 2024-12-09T23:58:57,352 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89b6654328f161896b9d470a5e0e55fb/A in TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:57,352 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89b6654328f161896b9d470a5e0e55fb/B in TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:57,352 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/c14e98b884d74b20aca964a1b1cd07e7, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/8b5ceea34c55443cb7f00f5c65a8bfec, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/743c280d45aa4d038ad8bdf59e2eec47, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/2c59f1d7c7854531bef87a2130fe21c6] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp, totalSize=51.8 K 2024-12-09T23:58:57,352 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/f8e7445498e848029d39914c06145d3a, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/e2f2225719004f9d9f3e8bf1367a69c9, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/e153ac1233d547459831267a8c171520, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/ba422969416b4f919090eef2270d0e1e] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp, totalSize=47.2 K 2024-12-09T23:58:57,353 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8e7445498e848029d39914c06145d3a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733788732586 2024-12-09T23:58:57,353 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting c14e98b884d74b20aca964a1b1cd07e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733788732586 2024-12-09T23:58:57,353 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2f2225719004f9d9f3e8bf1367a69c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733788733266 2024-12-09T23:58:57,353 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b5ceea34c55443cb7f00f5c65a8bfec, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733788733266 2024-12-09T23:58:57,353 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting e153ac1233d547459831267a8c171520, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733788734169 2024-12-09T23:58:57,353 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 743c280d45aa4d038ad8bdf59e2eec47, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733788734169 2024-12-09T23:58:57,353 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba422969416b4f919090eef2270d0e1e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733788735346 2024-12-09T23:58:57,353 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c59f1d7c7854531bef87a2130fe21c6, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733788735346 2024-12-09T23:58:57,361 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89b6654328f161896b9d470a5e0e55fb#B#compaction#357 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:57,361 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89b6654328f161896b9d470a5e0e55fb#A#compaction#358 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:57,361 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/cf9655e8201a4c9f835e5ed56193bf74 is 50, key is test_row_0/A:col10/1733788735355/Put/seqid=0 2024-12-09T23:58:57,361 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/aeae5294efef4d1e81efe0d6a2e71c65 is 50, key is test_row_0/B:col10/1733788735355/Put/seqid=0 2024-12-09T23:58:57,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742248_1424 (size=12493) 2024-12-09T23:58:57,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742249_1425 (size=12493) 2024-12-09T23:58:57,373 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/aeae5294efef4d1e81efe0d6a2e71c65 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/aeae5294efef4d1e81efe0d6a2e71c65 2024-12-09T23:58:57,377 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 89b6654328f161896b9d470a5e0e55fb/B of 89b6654328f161896b9d470a5e0e55fb into aeae5294efef4d1e81efe0d6a2e71c65(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:57,377 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:58:57,377 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., storeName=89b6654328f161896b9d470a5e0e55fb/B, priority=12, startTime=1733788737351; duration=0sec 2024-12-09T23:58:57,378 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:58:57,378 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89b6654328f161896b9d470a5e0e55fb:B 2024-12-09T23:58:57,378 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T23:58:57,379 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T23:58:57,379 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): 89b6654328f161896b9d470a5e0e55fb/C is initiating minor compaction (all files) 2024-12-09T23:58:57,380 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89b6654328f161896b9d470a5e0e55fb/C in TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:57,380 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/9930a9d5f49e4747842b5337a28cda8b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/fb215f3cb33f4119a25d2f1a911156e8, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/991021ce8d35487daf105b64c7dab6ef, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/6a6ecfa610b341cfae8f7f0ac156961d] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp, totalSize=47.2 K 2024-12-09T23:58:57,380 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9930a9d5f49e4747842b5337a28cda8b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733788732586 2024-12-09T23:58:57,380 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb215f3cb33f4119a25d2f1a911156e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733788733266 2024-12-09T23:58:57,380 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 991021ce8d35487daf105b64c7dab6ef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733788734169 2024-12-09T23:58:57,381 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a6ecfa610b341cfae8f7f0ac156961d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733788735346 2024-12-09T23:58:57,387 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89b6654328f161896b9d470a5e0e55fb#C#compaction#359 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:58:57,387 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/82094266df7c48769b01c4a7770742a3 is 50, key is test_row_0/C:col10/1733788735355/Put/seqid=0 2024-12-09T23:58:57,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742250_1426 (size=12493) 2024-12-09T23:58:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on 89b6654328f161896b9d470a5e0e55fb 2024-12-09T23:58:57,457 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 89b6654328f161896b9d470a5e0e55fb 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T23:58:57,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=A 2024-12-09T23:58:57,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:57,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=B 2024-12-09T23:58:57,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:57,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=C 2024-12-09T23:58:57,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:57,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/84e80156e38f4cb9898eb2ba92d4da4c is 50, key is test_row_0/A:col10/1733788736528/Put/seqid=0 2024-12-09T23:58:57,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742251_1427 (size=12147) 2024-12-09T23:58:57,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:57,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788797507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:57,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:57,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:57,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:57,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:57,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788797509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:57,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:57,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:57,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:57,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:57,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788797623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:57,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:57,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788797623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:57,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:57,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:57,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:57,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:57,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:57,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:57,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:57,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788797662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:57,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:57,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:57,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:57,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:57,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788797673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:57,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:57,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:57,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:57,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:57,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51356 deadline: 1733788797673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:57,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:57,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:57,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:57,771 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/cf9655e8201a4c9f835e5ed56193bf74 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/cf9655e8201a4c9f835e5ed56193bf74 2024-12-09T23:58:57,775 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 89b6654328f161896b9d470a5e0e55fb/A of 89b6654328f161896b9d470a5e0e55fb into cf9655e8201a4c9f835e5ed56193bf74(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:57,775 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:58:57,775 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., storeName=89b6654328f161896b9d470a5e0e55fb/A, priority=12, startTime=1733788737351; duration=0sec 2024-12-09T23:58:57,775 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:57,775 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89b6654328f161896b9d470a5e0e55fb:A 2024-12-09T23:58:57,793 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/82094266df7c48769b01c4a7770742a3 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/82094266df7c48769b01c4a7770742a3 2024-12-09T23:58:57,796 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 89b6654328f161896b9d470a5e0e55fb/C of 89b6654328f161896b9d470a5e0e55fb into 82094266df7c48769b01c4a7770742a3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:58:57,796 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:58:57,796 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., storeName=89b6654328f161896b9d470a5e0e55fb/C, priority=12, startTime=1733788737351; duration=0sec 2024-12-09T23:58:57,796 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:58:57,796 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89b6654328f161896b9d470a5e0e55fb:C 2024-12-09T23:58:57,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:57,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788797833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:57,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:57,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:57,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:57,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:57,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788797833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:57,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:57,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:57,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:57,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/84e80156e38f4cb9898eb2ba92d4da4c 2024-12-09T23:58:57,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/66a4e06e75e9408788b308dc6168e4e3 is 50, key is test_row_0/B:col10/1733788736528/Put/seqid=0 2024-12-09T23:58:57,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742252_1428 (size=9757) 2024-12-09T23:58:58,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:58,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788798143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:58,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:58,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:58,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:58,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:58,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788798143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:58,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:58,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:58,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:58,277 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/66a4e06e75e9408788b308dc6168e4e3 2024-12-09T23:58:58,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/447ff07340ac4ae9b5a68cd212ed05a9 is 50, key is test_row_0/C:col10/1733788736528/Put/seqid=0 2024-12-09T23:58:58,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742253_1429 (size=9757) 2024-12-09T23:58:58,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-09T23:58:58,303 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:58:58,303 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:58:58,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-09T23:58:58,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T23:58:58,305 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:58:58,305 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:58:58,305 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:58:58,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T23:58:58,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-09T23:58:58,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:58,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. as already flushing 2024-12-09T23:58:58,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:58,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:58,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:58,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:58,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-09T23:58:58,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:58,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. as already flushing 2024-12-09T23:58:58,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:58,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:58,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:58,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:58,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T23:58:58,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:58,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788798652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:58,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:58,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788798652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:58,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:58,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788798673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:58,677 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_0' in 89b6654328f161896b9d470a5e0e55fb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:58:58,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:58,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:58,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:58,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/447ff07340ac4ae9b5a68cd212ed05a9 2024-12-09T23:58:58,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:58,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51356 deadline: 1733788798683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:58,687 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_1' in 89b6654328f161896b9d470a5e0e55fb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:58:58,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:58,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:58,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:58,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/84e80156e38f4cb9898eb2ba92d4da4c as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/84e80156e38f4cb9898eb2ba92d4da4c 2024-12-09T23:58:58,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/84e80156e38f4cb9898eb2ba92d4da4c, entries=150, sequenceid=164, filesize=11.9 K 2024-12-09T23:58:58,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/66a4e06e75e9408788b308dc6168e4e3 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/66a4e06e75e9408788b308dc6168e4e3 2024-12-09T23:58:58,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:58,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/66a4e06e75e9408788b308dc6168e4e3, entries=100, sequenceid=164, filesize=9.5 K 2024-12-09T23:58:58,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788798692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:58,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/447ff07340ac4ae9b5a68cd212ed05a9 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/447ff07340ac4ae9b5a68cd212ed05a9 2024-12-09T23:58:58,697 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_2' in 89b6654328f161896b9d470a5e0e55fb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:58:58,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:58,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:58,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:58,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/447ff07340ac4ae9b5a68cd212ed05a9, entries=100, sequenceid=164, filesize=9.5 K 2024-12-09T23:58:58,701 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 89b6654328f161896b9d470a5e0e55fb in 1244ms, sequenceid=164, compaction requested=false 2024-12-09T23:58:58,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:58:58,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-09T23:58:58,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:58,762 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 89b6654328f161896b9d470a5e0e55fb 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T23:58:58,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=A 2024-12-09T23:58:58,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:58,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=B 2024-12-09T23:58:58,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:58,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=C 2024-12-09T23:58:58,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:58,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/588467211f2e4dd7bde979687a3cec99 is 50, key is test_row_0/A:col10/1733788737498/Put/seqid=0 2024-12-09T23:58:58,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742254_1430 (size=12151) 2024-12-09T23:58:58,771 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/588467211f2e4dd7bde979687a3cec99 2024-12-09T23:58:58,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/1d2760caf9d7460d92338d4eea3d2530 is 50, key is test_row_0/B:col10/1733788737498/Put/seqid=0 2024-12-09T23:58:58,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742255_1431 (size=12151) 2024-12-09T23:58:58,789 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/1d2760caf9d7460d92338d4eea3d2530 2024-12-09T23:58:58,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/da7863c110eb42c49c028c4bb81a97a6 is 50, key is test_row_0/C:col10/1733788737498/Put/seqid=0 2024-12-09T23:58:58,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742256_1432 (size=12151) 2024-12-09T23:58:58,800 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/da7863c110eb42c49c028c4bb81a97a6 2024-12-09T23:58:58,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/588467211f2e4dd7bde979687a3cec99 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/588467211f2e4dd7bde979687a3cec99 2024-12-09T23:58:58,807 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/588467211f2e4dd7bde979687a3cec99, entries=150, sequenceid=185, filesize=11.9 K 2024-12-09T23:58:58,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/1d2760caf9d7460d92338d4eea3d2530 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/1d2760caf9d7460d92338d4eea3d2530 2024-12-09T23:58:58,813 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/1d2760caf9d7460d92338d4eea3d2530, entries=150, sequenceid=185, filesize=11.9 K 2024-12-09T23:58:58,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/da7863c110eb42c49c028c4bb81a97a6 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/da7863c110eb42c49c028c4bb81a97a6 2024-12-09T23:58:58,817 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/da7863c110eb42c49c028c4bb81a97a6, entries=150, sequenceid=185, filesize=11.9 K 2024-12-09T23:58:58,817 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 89b6654328f161896b9d470a5e0e55fb in 55ms, sequenceid=185, compaction requested=true 2024-12-09T23:58:58,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:58:58,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:58,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-09T23:58:58,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-09T23:58:58,821 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-09T23:58:58,821 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 513 msec 2024-12-09T23:58:58,822 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 518 msec 2024-12-09T23:58:58,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T23:58:58,933 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:58:58,934 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:58:58,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-09T23:58:58,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T23:58:58,936 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:58:58,938 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:58:58,938 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:58:59,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T23:58:59,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-09T23:58:59,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:59,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:58:59,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:59,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-09T23:58:59,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-09T23:58:59,095 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-09T23:58:59,095 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-12-09T23:58:59,096 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 161 msec 2024-12-09T23:58:59,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T23:58:59,253 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:58:59,253 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:58:59,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-09T23:58:59,254 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:58:59,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T23:58:59,255 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:58:59,255 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:58:59,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T23:58:59,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-09T23:58:59,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:59,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:58:59,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:59,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-09T23:58:59,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-09T23:58:59,411 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-09T23:58:59,411 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-12-09T23:58:59,413 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 159 msec 2024-12-09T23:58:59,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T23:58:59,573 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:58:59,573 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:58:59,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-09T23:58:59,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-09T23:58:59,574 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:58:59,575 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:58:59,575 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:58:59,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on 89b6654328f161896b9d470a5e0e55fb 2024-12-09T23:58:59,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 89b6654328f161896b9d470a5e0e55fb 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T23:58:59,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=A 2024-12-09T23:58:59,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:59,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=B 2024-12-09T23:58:59,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:59,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=C 2024-12-09T23:58:59,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:58:59,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/c3f3c392115a41bbbf41587af5c69abd is 50, key is test_row_0/A:col10/1733788739676/Put/seqid=0 2024-12-09T23:58:59,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-09T23:58:59,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742257_1433 (size=14541) 2024-12-09T23:58:59,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/c3f3c392115a41bbbf41587af5c69abd 2024-12-09T23:58:59,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/956632b4262643fc8723ec23da37a70f is 50, key is test_row_0/B:col10/1733788739676/Put/seqid=0 2024-12-09T23:58:59,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742258_1434 (size=12151) 2024-12-09T23:58:59,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-09T23:58:59,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:59,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. as already flushing 2024-12-09T23:58:59,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:59,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:59,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:59,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:59,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733788739723, max timestamp=1733788739745Num uniques 859; 2024-12-09T23:58:59,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T23:58:59,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733788739723, max timestamp=1733788739745Num uniques 859; 2024-12-09T23:58:59,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T23:58:59,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733788739723, max timestamp=1733788739745Num uniques 859; 2024-12-09T23:58:59,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T23:58:59,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:59,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788799799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:59,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:59,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:59,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788799801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:59,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:59,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:59,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:59,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:59,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:59,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-09T23:58:59,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:59,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. as already flushing 2024-12-09T23:58:59,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:58:59,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:59,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:59,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:58:59,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-09T23:58:59,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:59,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788799912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:59,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:58:59,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:59,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:59,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788799913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:58:59,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:58:59,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:59,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:58:59,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:00,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-09T23:59:00,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:00,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. as already flushing 2024-12-09T23:59:00,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:00,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:00,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:00,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:00,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/956632b4262643fc8723ec23da37a70f 2024-12-09T23:59:00,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/caa2d7e7b0a24ec08d14902be4ff9500 is 50, key is test_row_0/C:col10/1733788739676/Put/seqid=0 2024-12-09T23:59:00,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742259_1435 (size=12151) 2024-12-09T23:59:00,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788800123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:00,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:00,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:00,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:00,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788800123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:00,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:00,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:00,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:00,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-09T23:59:00,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:00,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. as already flushing 2024-12-09T23:59:00,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:00,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:00,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:00,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-09T23:59:00,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-09T23:59:00,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:00,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. as already flushing 2024-12-09T23:59:00,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:00,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:00,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:00,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:00,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:00,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788800433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:00,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:00,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:00,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:00,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788800433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:00,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:00,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:00,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:00,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:00,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-09T23:59:00,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:00,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. as already flushing 2024-12-09T23:59:00,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:00,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:00,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:00,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:00,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/caa2d7e7b0a24ec08d14902be4ff9500 2024-12-09T23:59:00,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/c3f3c392115a41bbbf41587af5c69abd as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/c3f3c392115a41bbbf41587af5c69abd 2024-12-09T23:59:00,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/c3f3c392115a41bbbf41587af5c69abd, entries=200, sequenceid=197, filesize=14.2 K 2024-12-09T23:59:00,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/956632b4262643fc8723ec23da37a70f as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/956632b4262643fc8723ec23da37a70f 2024-12-09T23:59:00,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/956632b4262643fc8723ec23da37a70f, entries=150, sequenceid=197, filesize=11.9 K 2024-12-09T23:59:00,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/caa2d7e7b0a24ec08d14902be4ff9500 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/caa2d7e7b0a24ec08d14902be4ff9500 2024-12-09T23:59:00,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/caa2d7e7b0a24ec08d14902be4ff9500, entries=150, sequenceid=197, filesize=11.9 K 2024-12-09T23:59:00,516 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 89b6654328f161896b9d470a5e0e55fb in 839ms, sequenceid=197, compaction requested=true 2024-12-09T23:59:00,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:59:00,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89b6654328f161896b9d470a5e0e55fb:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:59:00,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:59:00,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89b6654328f161896b9d470a5e0e55fb:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:59:00,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:59:00,516 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T23:59:00,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89b6654328f161896b9d470a5e0e55fb:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:59:00,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:59:00,517 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T23:59:00,517 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T23:59:00,517 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): 89b6654328f161896b9d470a5e0e55fb/B is initiating minor compaction (all files) 2024-12-09T23:59:00,518 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89b6654328f161896b9d470a5e0e55fb/B in TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:00,518 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/aeae5294efef4d1e81efe0d6a2e71c65, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/66a4e06e75e9408788b308dc6168e4e3, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/1d2760caf9d7460d92338d4eea3d2530, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/956632b4262643fc8723ec23da37a70f] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp, totalSize=45.5 K 2024-12-09T23:59:00,518 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting aeae5294efef4d1e81efe0d6a2e71c65, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733788735346 2024-12-09T23:59:00,518 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 66a4e06e75e9408788b308dc6168e4e3, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733788736528 2024-12-09T23:59:00,518 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d2760caf9d7460d92338d4eea3d2530, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733788737498 2024-12-09T23:59:00,519 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 956632b4262643fc8723ec23da37a70f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733788739672 2024-12-09T23:59:00,524 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89b6654328f161896b9d470a5e0e55fb#B#compaction#369 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:59:00,525 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/8deef2bf19d5429684e3da3eae0469bb is 50, key is test_row_0/B:col10/1733788739676/Put/seqid=0 2024-12-09T23:59:00,527 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51332 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T23:59:00,527 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): 89b6654328f161896b9d470a5e0e55fb/A is initiating minor compaction (all files) 2024-12-09T23:59:00,527 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89b6654328f161896b9d470a5e0e55fb/A in TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:00,527 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/cf9655e8201a4c9f835e5ed56193bf74, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/84e80156e38f4cb9898eb2ba92d4da4c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/588467211f2e4dd7bde979687a3cec99, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/c3f3c392115a41bbbf41587af5c69abd] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp, totalSize=50.1 K 2024-12-09T23:59:00,527 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf9655e8201a4c9f835e5ed56193bf74, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733788735346 2024-12-09T23:59:00,528 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84e80156e38f4cb9898eb2ba92d4da4c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733788736520 2024-12-09T23:59:00,528 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 588467211f2e4dd7bde979687a3cec99, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733788737498 2024-12-09T23:59:00,528 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3f3c392115a41bbbf41587af5c69abd, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733788739672 2024-12-09T23:59:00,540 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89b6654328f161896b9d470a5e0e55fb#A#compaction#370 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:59:00,541 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/34c00421a06b476480d19f70ddcb377e is 50, key is test_row_0/A:col10/1733788739676/Put/seqid=0 2024-12-09T23:59:00,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742260_1436 (size=12629) 2024-12-09T23:59:00,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742261_1437 (size=12629) 2024-12-09T23:59:00,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-09T23:59:00,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:00,642 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 89b6654328f161896b9d470a5e0e55fb 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-09T23:59:00,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=A 2024-12-09T23:59:00,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:59:00,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=B 2024-12-09T23:59:00,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:59:00,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=C 2024-12-09T23:59:00,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:59:00,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/fcbaf39f397248bea0549ba11ffce3ff is 50, key is test_row_0/A:col10/1733788739800/Put/seqid=0 2024-12-09T23:59:00,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742262_1438 (size=12151) 2024-12-09T23:59:00,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on 89b6654328f161896b9d470a5e0e55fb 2024-12-09T23:59:00,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. as already flushing 2024-12-09T23:59:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-09T23:59:00,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:00,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51356 deadline: 1733788800726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:00,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:00,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:00,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:00,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:00,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788800729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:00,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:00,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:00,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:00,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:00,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788800730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:00,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:00,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:00,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:00,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:00,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51356 deadline: 1733788800832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:00,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:00,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:00,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:00,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:00,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788800842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:00,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:00,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:00,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:00,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788800842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:00,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:00,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:00,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:00,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:00,949 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/8deef2bf19d5429684e3da3eae0469bb as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/8deef2bf19d5429684e3da3eae0469bb 2024-12-09T23:59:00,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:00,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788800942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:00,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:00,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:00,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:00,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:00,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788800942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:00,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:00,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:00,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:00,955 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 89b6654328f161896b9d470a5e0e55fb/B of 89b6654328f161896b9d470a5e0e55fb into 8deef2bf19d5429684e3da3eae0469bb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:59:00,955 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:59:00,955 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., storeName=89b6654328f161896b9d470a5e0e55fb/B, priority=12, startTime=1733788740516; duration=0sec 2024-12-09T23:59:00,955 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:59:00,955 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89b6654328f161896b9d470a5e0e55fb:B 2024-12-09T23:59:00,955 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T23:59:00,956 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T23:59:00,956 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): 89b6654328f161896b9d470a5e0e55fb/C is initiating minor compaction (all files) 2024-12-09T23:59:00,957 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89b6654328f161896b9d470a5e0e55fb/C in TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:00,957 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/82094266df7c48769b01c4a7770742a3, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/447ff07340ac4ae9b5a68cd212ed05a9, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/da7863c110eb42c49c028c4bb81a97a6, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/caa2d7e7b0a24ec08d14902be4ff9500] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp, totalSize=45.5 K 2024-12-09T23:59:00,957 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 82094266df7c48769b01c4a7770742a3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733788735346 2024-12-09T23:59:00,957 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 447ff07340ac4ae9b5a68cd212ed05a9, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733788736528 2024-12-09T23:59:00,957 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting da7863c110eb42c49c028c4bb81a97a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733788737498 2024-12-09T23:59:00,958 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting caa2d7e7b0a24ec08d14902be4ff9500, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733788739672 2024-12-09T23:59:00,959 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/34c00421a06b476480d19f70ddcb377e as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/34c00421a06b476480d19f70ddcb377e 2024-12-09T23:59:00,963 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 89b6654328f161896b9d470a5e0e55fb/A of 89b6654328f161896b9d470a5e0e55fb into 34c00421a06b476480d19f70ddcb377e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:59:00,963 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:59:00,965 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., storeName=89b6654328f161896b9d470a5e0e55fb/A, priority=12, startTime=1733788740516; duration=0sec 2024-12-09T23:59:00,965 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:59:00,965 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89b6654328f161896b9d470a5e0e55fb:A 2024-12-09T23:59:00,966 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89b6654328f161896b9d470a5e0e55fb#C#compaction#372 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:59:00,966 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/e60e19d9adb845429abab94f0beffca4 is 50, key is test_row_0/C:col10/1733788739676/Put/seqid=0 2024-12-09T23:59:00,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742263_1439 (size=12629) 2024-12-09T23:59:01,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:01,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51356 deadline: 1733788801042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:01,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:01,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:01,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:01,051 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/fcbaf39f397248bea0549ba11ffce3ff 2024-12-09T23:59:01,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/886f73fae48e414a84623724354865f8 is 50, key is test_row_0/B:col10/1733788739800/Put/seqid=0 2024-12-09T23:59:01,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742264_1440 (size=12151) 2024-12-09T23:59:01,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:01,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788801062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:01,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:01,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:01,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:01,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:01,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788801062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:01,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:01,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:01,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:01,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:01,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51356 deadline: 1733788801363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:01,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:01,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:01,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:01,374 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/e60e19d9adb845429abab94f0beffca4 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/e60e19d9adb845429abab94f0beffca4 2024-12-09T23:59:01,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:01,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788801373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:01,376 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:01,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:01,376 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:01,376 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:01,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788801373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:01,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:01,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:01,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:01,378 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 89b6654328f161896b9d470a5e0e55fb/C of 89b6654328f161896b9d470a5e0e55fb into e60e19d9adb845429abab94f0beffca4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:59:01,378 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:59:01,378 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., storeName=89b6654328f161896b9d470a5e0e55fb/C, priority=12, startTime=1733788740516; duration=0sec 2024-12-09T23:59:01,378 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:59:01,378 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89b6654328f161896b9d470a5e0e55fb:C 2024-12-09T23:59:01,461 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/886f73fae48e414a84623724354865f8 2024-12-09T23:59:01,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/a1521de89e0b44ffbe6860cf8a3526b7 is 50, key is test_row_0/C:col10/1733788739800/Put/seqid=0 2024-12-09T23:59:01,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742265_1441 (size=12151) 2024-12-09T23:59:01,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-09T23:59:01,870 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/a1521de89e0b44ffbe6860cf8a3526b7 2024-12-09T23:59:01,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/fcbaf39f397248bea0549ba11ffce3ff as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/fcbaf39f397248bea0549ba11ffce3ff 2024-12-09T23:59:01,877 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/fcbaf39f397248bea0549ba11ffce3ff, entries=150, sequenceid=233, filesize=11.9 K 2024-12-09T23:59:01,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/886f73fae48e414a84623724354865f8 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/886f73fae48e414a84623724354865f8 2024-12-09T23:59:01,880 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/886f73fae48e414a84623724354865f8, entries=150, sequenceid=233, filesize=11.9 K 2024-12-09T23:59:01,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:01,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51356 deadline: 1733788801873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:01,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:01,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:01,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:01,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/a1521de89e0b44ffbe6860cf8a3526b7 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/a1521de89e0b44ffbe6860cf8a3526b7 2024-12-09T23:59:01,884 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/a1521de89e0b44ffbe6860cf8a3526b7, entries=150, sequenceid=233, filesize=11.9 K 2024-12-09T23:59:01,885 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 89b6654328f161896b9d470a5e0e55fb in 1243ms, sequenceid=233, compaction requested=false 2024-12-09T23:59:01,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:59:01,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:01,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-09T23:59:01,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-09T23:59:01,887 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-09T23:59:01,887 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3110 sec 2024-12-09T23:59:01,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on 89b6654328f161896b9d470a5e0e55fb 2024-12-09T23:59:01,888 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 89b6654328f161896b9d470a5e0e55fb 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T23:59:01,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=A 2024-12-09T23:59:01,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:01,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=B 2024-12-09T23:59:01,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:01,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=C 2024-12-09T23:59:01,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:01,890 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.3150 sec 2024-12-09T23:59:01,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/634507fbbac34b9ea23fba6fd82e037b is 50, key is test_row_0/A:col10/1733788740706/Put/seqid=0 2024-12-09T23:59:01,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742266_1442 (size=14541) 2024-12-09T23:59:01,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733788741922, max timestamp=1733788741945Num uniques 859; 2024-12-09T23:59:01,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T23:59:01,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733788741922, max timestamp=1733788741945Num uniques 859; 2024-12-09T23:59:01,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733788741922, max timestamp=1733788741945Num uniques 859; 2024-12-09T23:59:01,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T23:59:01,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T23:59:01,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:01,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788801977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:01,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:01,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:01,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:01,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:01,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788801982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:01,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:01,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788801983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:01,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:01,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:01,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:01,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:01,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788801983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:01,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:01,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:01,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:02,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:02,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788802093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:02,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:02,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:02,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:02,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788802093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:02,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:02,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:02,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:02,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788802093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:02,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:02,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:02,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:02,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:02,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:02,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:02,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788802102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:02,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:02,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:02,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:02,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/634507fbbac34b9ea23fba6fd82e037b 2024-12-09T23:59:02,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:02,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788802303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:02,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:02,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:02,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788802303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:02,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:02,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:02,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:02,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:02,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:02,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:02,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788802303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:02,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:02,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:02,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:02,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:02,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788802312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:02,319 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:02,319 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:02,319 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:02,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/bda901fa1be24a3c9a12474160da9832 is 50, key is test_row_0/B:col10/1733788740706/Put/seqid=0 2024-12-09T23:59:02,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742267_1443 (size=12151) 2024-12-09T23:59:02,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:02,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788802612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:02,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:02,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:02,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788802613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:02,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:02,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:02,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:02,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:02,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:02,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:02,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788802613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:02,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:02,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:02,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:02,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:02,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788802623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:02,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:02,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:02,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:02,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/bda901fa1be24a3c9a12474160da9832 2024-12-09T23:59:02,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/ca1a679f3f6d428d9f6cf0a160060748 is 50, key is test_row_0/C:col10/1733788740706/Put/seqid=0 2024-12-09T23:59:02,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742268_1444 (size=12151) 2024-12-09T23:59:02,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:02,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51356 deadline: 1733788802893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:02,897 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_1' in 89b6654328f161896b9d470a5e0e55fb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:59:02,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:02,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:02,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:03,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:03,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788803123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:03,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:03,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:03,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:03,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788803123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:03,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:03,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:03,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:03,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:03,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:03,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:03,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788803133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:03,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788803133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:03,140 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:03,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:03,140 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:03,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:03,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:03,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:03,164 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/ca1a679f3f6d428d9f6cf0a160060748 2024-12-09T23:59:03,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/634507fbbac34b9ea23fba6fd82e037b as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/634507fbbac34b9ea23fba6fd82e037b 2024-12-09T23:59:03,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/634507fbbac34b9ea23fba6fd82e037b, entries=200, sequenceid=247, filesize=14.2 K 2024-12-09T23:59:03,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/bda901fa1be24a3c9a12474160da9832 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/bda901fa1be24a3c9a12474160da9832 2024-12-09T23:59:03,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/bda901fa1be24a3c9a12474160da9832, entries=150, sequenceid=247, filesize=11.9 K 2024-12-09T23:59:03,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/ca1a679f3f6d428d9f6cf0a160060748 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/ca1a679f3f6d428d9f6cf0a160060748 2024-12-09T23:59:03,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/ca1a679f3f6d428d9f6cf0a160060748, entries=150, sequenceid=247, filesize=11.9 K 2024-12-09T23:59:03,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 89b6654328f161896b9d470a5e0e55fb in 1299ms, sequenceid=247, compaction requested=true 2024-12-09T23:59:03,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:59:03,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89b6654328f161896b9d470a5e0e55fb:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:59:03,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:59:03,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89b6654328f161896b9d470a5e0e55fb:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:59:03,187 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:59:03,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:59:03,187 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:59:03,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89b6654328f161896b9d470a5e0e55fb:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:59:03,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:59:03,188 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39321 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:59:03,188 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:59:03,188 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): 89b6654328f161896b9d470a5e0e55fb/A is initiating minor compaction (all files) 2024-12-09T23:59:03,188 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): 89b6654328f161896b9d470a5e0e55fb/B is initiating minor compaction (all files) 2024-12-09T23:59:03,188 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89b6654328f161896b9d470a5e0e55fb/B in TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:03,188 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89b6654328f161896b9d470a5e0e55fb/A in TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:03,188 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/8deef2bf19d5429684e3da3eae0469bb, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/886f73fae48e414a84623724354865f8, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/bda901fa1be24a3c9a12474160da9832] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp, totalSize=36.1 K 2024-12-09T23:59:03,188 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/34c00421a06b476480d19f70ddcb377e, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/fcbaf39f397248bea0549ba11ffce3ff, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/634507fbbac34b9ea23fba6fd82e037b] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp, totalSize=38.4 K 2024-12-09T23:59:03,188 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 34c00421a06b476480d19f70ddcb377e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733788739672 2024-12-09T23:59:03,189 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8deef2bf19d5429684e3da3eae0469bb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733788739672 2024-12-09T23:59:03,189 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting fcbaf39f397248bea0549ba11ffce3ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733788739769 2024-12-09T23:59:03,189 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 886f73fae48e414a84623724354865f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733788739769 2024-12-09T23:59:03,189 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 634507fbbac34b9ea23fba6fd82e037b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733788740706 2024-12-09T23:59:03,189 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting bda901fa1be24a3c9a12474160da9832, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733788740706 2024-12-09T23:59:03,197 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89b6654328f161896b9d470a5e0e55fb#A#compaction#378 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:59:03,197 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89b6654328f161896b9d470a5e0e55fb#B#compaction#379 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:59:03,197 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/d7f0f470a0264e39b9fd206fac059f49 is 50, key is test_row_0/B:col10/1733788740706/Put/seqid=0 2024-12-09T23:59:03,197 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/627ddb1c6671457da4310d1b7566a61b is 50, key is test_row_0/A:col10/1733788740706/Put/seqid=0 2024-12-09T23:59:03,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742270_1446 (size=12731) 2024-12-09T23:59:03,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742269_1445 (size=12731) 2024-12-09T23:59:03,218 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/627ddb1c6671457da4310d1b7566a61b as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/627ddb1c6671457da4310d1b7566a61b 2024-12-09T23:59:03,221 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89b6654328f161896b9d470a5e0e55fb/A of 89b6654328f161896b9d470a5e0e55fb into 627ddb1c6671457da4310d1b7566a61b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:59:03,222 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:59:03,222 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., storeName=89b6654328f161896b9d470a5e0e55fb/A, priority=13, startTime=1733788743187; duration=0sec 2024-12-09T23:59:03,222 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:59:03,222 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89b6654328f161896b9d470a5e0e55fb:A 2024-12-09T23:59:03,222 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:59:03,222 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:59:03,222 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): 89b6654328f161896b9d470a5e0e55fb/C is initiating minor compaction (all files) 2024-12-09T23:59:03,222 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89b6654328f161896b9d470a5e0e55fb/C in TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:03,223 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/e60e19d9adb845429abab94f0beffca4, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/a1521de89e0b44ffbe6860cf8a3526b7, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/ca1a679f3f6d428d9f6cf0a160060748] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp, totalSize=36.1 K 2024-12-09T23:59:03,223 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting e60e19d9adb845429abab94f0beffca4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733788739672 2024-12-09T23:59:03,223 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting a1521de89e0b44ffbe6860cf8a3526b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733788739769 2024-12-09T23:59:03,223 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting ca1a679f3f6d428d9f6cf0a160060748, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733788740706 2024-12-09T23:59:03,229 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89b6654328f161896b9d470a5e0e55fb#C#compaction#380 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:59:03,229 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/5f8bf24cb421495098feb2b67d83e178 is 50, key is test_row_0/C:col10/1733788740706/Put/seqid=0 2024-12-09T23:59:03,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742271_1447 (size=12731) 2024-12-09T23:59:03,237 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/5f8bf24cb421495098feb2b67d83e178 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/5f8bf24cb421495098feb2b67d83e178 2024-12-09T23:59:03,241 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89b6654328f161896b9d470a5e0e55fb/C of 89b6654328f161896b9d470a5e0e55fb into 5f8bf24cb421495098feb2b67d83e178(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:59:03,241 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:59:03,241 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., storeName=89b6654328f161896b9d470a5e0e55fb/C, priority=13, startTime=1733788743187; duration=0sec 2024-12-09T23:59:03,241 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:59:03,241 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89b6654328f161896b9d470a5e0e55fb:C 2024-12-09T23:59:03,615 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/d7f0f470a0264e39b9fd206fac059f49 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/d7f0f470a0264e39b9fd206fac059f49 2024-12-09T23:59:03,618 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89b6654328f161896b9d470a5e0e55fb/B of 89b6654328f161896b9d470a5e0e55fb into d7f0f470a0264e39b9fd206fac059f49(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:59:03,618 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:59:03,618 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., storeName=89b6654328f161896b9d470a5e0e55fb/B, priority=13, startTime=1733788743187; duration=0sec 2024-12-09T23:59:03,618 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:59:03,618 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89b6654328f161896b9d470a5e0e55fb:B 2024-12-09T23:59:03,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-09T23:59:03,743 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:59:03,743 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:59:03,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-09T23:59:03,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T23:59:03,745 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:59:03,745 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:59:03,746 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:59:03,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T23:59:03,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T23:59:03,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:03,897 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 89b6654328f161896b9d470a5e0e55fb 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-09T23:59:03,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=A 2024-12-09T23:59:03,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:59:03,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=B 2024-12-09T23:59:03,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:59:03,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=C 2024-12-09T23:59:03,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:59:03,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/efac0c882fec419cbf29f430b57418dd is 50, key is test_row_0/A:col10/1733788741982/Put/seqid=0 2024-12-09T23:59:03,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742272_1448 (size=12301) 2024-12-09T23:59:04,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T23:59:04,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on 89b6654328f161896b9d470a5e0e55fb 2024-12-09T23:59:04,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. as already flushing 2024-12-09T23:59:04,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:04,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788804156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:04,162 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,162 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,162 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:04,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:04,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788804157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:04,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:04,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:04,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788804159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:04,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:04,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:04,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788804162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:04,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:04,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:04,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788804272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:04,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:04,276 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,276 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788804272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:04,276 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:04,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:04,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:04,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788804273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:04,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:04,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:04,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788804273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:04,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:04,305 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/efac0c882fec419cbf29f430b57418dd 2024-12-09T23:59:04,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/0a660c8ce53240beb4679f8c2afe2e9d is 50, key is test_row_0/B:col10/1733788741982/Put/seqid=0 2024-12-09T23:59:04,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742273_1449 (size=12301) 2024-12-09T23:59:04,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T23:59:04,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:04,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788804482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:04,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:04,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788804482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:04,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:04,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:04,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788804482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:04,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788804482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:04,716 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/0a660c8ce53240beb4679f8c2afe2e9d 2024-12-09T23:59:04,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/d73607d1f6f24d8993615b5f21de18fe is 50, key is test_row_0/C:col10/1733788741982/Put/seqid=0 2024-12-09T23:59:04,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742274_1450 (size=12301) 2024-12-09T23:59:04,727 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/d73607d1f6f24d8993615b5f21de18fe 2024-12-09T23:59:04,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/efac0c882fec419cbf29f430b57418dd as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/efac0c882fec419cbf29f430b57418dd 2024-12-09T23:59:04,733 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/efac0c882fec419cbf29f430b57418dd, entries=150, sequenceid=285, filesize=12.0 K 2024-12-09T23:59:04,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/0a660c8ce53240beb4679f8c2afe2e9d as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/0a660c8ce53240beb4679f8c2afe2e9d 2024-12-09T23:59:04,737 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/0a660c8ce53240beb4679f8c2afe2e9d, entries=150, sequenceid=285, filesize=12.0 K 2024-12-09T23:59:04,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/d73607d1f6f24d8993615b5f21de18fe as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/d73607d1f6f24d8993615b5f21de18fe 2024-12-09T23:59:04,741 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/d73607d1f6f24d8993615b5f21de18fe, entries=150, sequenceid=285, filesize=12.0 K 2024-12-09T23:59:04,742 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 89b6654328f161896b9d470a5e0e55fb in 844ms, sequenceid=285, compaction requested=false 2024-12-09T23:59:04,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:59:04,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:04,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-09T23:59:04,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-09T23:59:04,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-09T23:59:04,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 997 msec 2024-12-09T23:59:04,746 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.0020 sec 2024-12-09T23:59:04,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on 89b6654328f161896b9d470a5e0e55fb 2024-12-09T23:59:04,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 89b6654328f161896b9d470a5e0e55fb 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T23:59:04,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=A 2024-12-09T23:59:04,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:04,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=B 2024-12-09T23:59:04,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:04,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=C 2024-12-09T23:59:04,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:04,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/d4e24f32cac542e5979c7d781269ab19 is 50, key is test_row_0/A:col10/1733788744161/Put/seqid=0 2024-12-09T23:59:04,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742275_1451 (size=17181) 2024-12-09T23:59:04,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:04,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788804824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:04,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:04,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:04,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788804824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:04,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:04,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:04,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788804825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:04,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:04,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788804826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:04,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:04,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:04,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T23:59:04,882 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:59:04,883 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:59:04,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-12-09T23:59:04,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-09T23:59:04,884 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:59:04,885 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:59:04,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:59:04,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:04,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51356 deadline: 1733788804923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:04,927 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_1' in 89b6654328f161896b9d470a5e0e55fb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:59:04,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:04,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:04,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788804933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:04,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:04,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:04,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788804943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:04,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788804943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:04,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788804943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:04,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-09T23:59:05,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T23:59:05,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:05,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. as already flushing 2024-12-09T23:59:05,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:05,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:05,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:05,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:05,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:05,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788805143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:05,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:05,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:05,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:05,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:05,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788805153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:05,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:05,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:05,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:05,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:05,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788805153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:05,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:05,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:05,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:05,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:05,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788805153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:05,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:05,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:05,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:05,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T23:59:05,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:05,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. as already flushing 2024-12-09T23:59:05,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:05,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:05,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:05,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:05,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-09T23:59:05,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/d4e24f32cac542e5979c7d781269ab19 2024-12-09T23:59:05,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/1abecd15dc934d17b261a60bb6ceddce is 50, key is test_row_0/B:col10/1733788744161/Put/seqid=0 2024-12-09T23:59:05,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742276_1452 (size=12301) 2024-12-09T23:59:05,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T23:59:05,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:05,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. as already flushing 2024-12-09T23:59:05,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:05,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:05,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:05,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:05,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:05,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788805452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:05,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:05,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:05,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:05,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:05,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788805463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:05,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:05,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:05,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:05,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:05,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788805463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:05,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:05,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:05,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:05,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:05,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788805463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:05,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:05,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:05,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:05,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T23:59:05,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:05,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. as already flushing 2024-12-09T23:59:05,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:05,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:05,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:05,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:05,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-09T23:59:05,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/1abecd15dc934d17b261a60bb6ceddce 2024-12-09T23:59:05,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/5c163393f2a14930bbd05c967d1b953c is 50, key is test_row_0/C:col10/1733788744161/Put/seqid=0 2024-12-09T23:59:05,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742277_1453 (size=12301) 2024-12-09T23:59:05,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T23:59:05,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:05,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. as already flushing 2024-12-09T23:59:05,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:05,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:05,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:05,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:05,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T23:59:05,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:05,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. as already flushing 2024-12-09T23:59:05,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:05,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:05,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:05,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:05,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T23:59:05,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:05,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. as already flushing 2024-12-09T23:59:05,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:05,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:05,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:05,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788805973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:05,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:05,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:05,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:05,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788805983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:05,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:05,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:05,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788805983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:05,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:05,989 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:05,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:05,989 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788805983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-09T23:59:06,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/5c163393f2a14930bbd05c967d1b953c 2024-12-09T23:59:06,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/d4e24f32cac542e5979c7d781269ab19 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/d4e24f32cac542e5979c7d781269ab19 2024-12-09T23:59:06,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/d4e24f32cac542e5979c7d781269ab19, entries=250, sequenceid=298, filesize=16.8 K 2024-12-09T23:59:06,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/1abecd15dc934d17b261a60bb6ceddce as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/1abecd15dc934d17b261a60bb6ceddce 2024-12-09T23:59:06,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/1abecd15dc934d17b261a60bb6ceddce, entries=150, sequenceid=298, filesize=12.0 K 2024-12-09T23:59:06,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/5c163393f2a14930bbd05c967d1b953c as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/5c163393f2a14930bbd05c967d1b953c 2024-12-09T23:59:06,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/5c163393f2a14930bbd05c967d1b953c, entries=150, sequenceid=298, filesize=12.0 K 2024-12-09T23:59:06,049 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 89b6654328f161896b9d470a5e0e55fb in 1253ms, sequenceid=298, compaction requested=true 2024-12-09T23:59:06,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:59:06,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89b6654328f161896b9d470a5e0e55fb:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:59:06,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:59:06,049 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:59:06,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89b6654328f161896b9d470a5e0e55fb:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:59:06,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:59:06,049 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:59:06,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89b6654328f161896b9d470a5e0e55fb:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:59:06,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:59:06,050 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:59:06,050 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42213 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:59:06,050 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): 89b6654328f161896b9d470a5e0e55fb/A is initiating minor compaction (all files) 2024-12-09T23:59:06,050 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): 89b6654328f161896b9d470a5e0e55fb/B is initiating minor compaction (all files) 2024-12-09T23:59:06,050 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89b6654328f161896b9d470a5e0e55fb/B in TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:06,050 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89b6654328f161896b9d470a5e0e55fb/A in TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:06,050 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/d7f0f470a0264e39b9fd206fac059f49, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/0a660c8ce53240beb4679f8c2afe2e9d, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/1abecd15dc934d17b261a60bb6ceddce] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp, totalSize=36.5 K 2024-12-09T23:59:06,050 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/627ddb1c6671457da4310d1b7566a61b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/efac0c882fec419cbf29f430b57418dd, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/d4e24f32cac542e5979c7d781269ab19] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp, totalSize=41.2 K 2024-12-09T23:59:06,051 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting d7f0f470a0264e39b9fd206fac059f49, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733788740706 2024-12-09T23:59:06,051 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 627ddb1c6671457da4310d1b7566a61b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733788740706 2024-12-09T23:59:06,051 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a660c8ce53240beb4679f8c2afe2e9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733788741981 2024-12-09T23:59:06,051 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting efac0c882fec419cbf29f430b57418dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733788741981 2024-12-09T23:59:06,051 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 1abecd15dc934d17b261a60bb6ceddce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733788744156 2024-12-09T23:59:06,051 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4e24f32cac542e5979c7d781269ab19, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733788744156 2024-12-09T23:59:06,057 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89b6654328f161896b9d470a5e0e55fb#A#compaction#388 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:59:06,057 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/aa0d74303db140cfa7230b9d00f99aa7 is 50, key is test_row_0/A:col10/1733788744161/Put/seqid=0 2024-12-09T23:59:06,057 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89b6654328f161896b9d470a5e0e55fb#B#compaction#387 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:59:06,058 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/e6235512ae4d44f18d7eb244b8394c51 is 50, key is test_row_0/B:col10/1733788744161/Put/seqid=0 2024-12-09T23:59:06,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742278_1454 (size=12983) 2024-12-09T23:59:06,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742279_1455 (size=12983) 2024-12-09T23:59:06,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T23:59:06,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:06,105 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 89b6654328f161896b9d470a5e0e55fb 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T23:59:06,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=A 2024-12-09T23:59:06,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:06,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=B 2024-12-09T23:59:06,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:06,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=C 2024-12-09T23:59:06,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:06,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/04a787bf57694835a8e977c83b5913f3 is 50, key is test_row_0/A:col10/1733788744823/Put/seqid=0 2024-12-09T23:59:06,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742280_1456 (size=12301) 2024-12-09T23:59:06,467 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/e6235512ae4d44f18d7eb244b8394c51 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/e6235512ae4d44f18d7eb244b8394c51 2024-12-09T23:59:06,467 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/aa0d74303db140cfa7230b9d00f99aa7 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/aa0d74303db140cfa7230b9d00f99aa7 2024-12-09T23:59:06,471 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89b6654328f161896b9d470a5e0e55fb/B of 89b6654328f161896b9d470a5e0e55fb into e6235512ae4d44f18d7eb244b8394c51(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:59:06,471 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89b6654328f161896b9d470a5e0e55fb/A of 89b6654328f161896b9d470a5e0e55fb into aa0d74303db140cfa7230b9d00f99aa7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:59:06,471 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:59:06,471 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:59:06,471 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., storeName=89b6654328f161896b9d470a5e0e55fb/B, priority=13, startTime=1733788746049; duration=0sec 2024-12-09T23:59:06,471 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., storeName=89b6654328f161896b9d470a5e0e55fb/A, priority=13, startTime=1733788746049; duration=0sec 2024-12-09T23:59:06,471 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:59:06,471 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:59:06,471 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89b6654328f161896b9d470a5e0e55fb:A 2024-12-09T23:59:06,471 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89b6654328f161896b9d470a5e0e55fb:B 2024-12-09T23:59:06,471 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:59:06,472 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:59:06,472 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): 89b6654328f161896b9d470a5e0e55fb/C is initiating minor compaction (all files) 2024-12-09T23:59:06,472 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89b6654328f161896b9d470a5e0e55fb/C in TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:06,472 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/5f8bf24cb421495098feb2b67d83e178, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/d73607d1f6f24d8993615b5f21de18fe, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/5c163393f2a14930bbd05c967d1b953c] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp, totalSize=36.5 K 2024-12-09T23:59:06,472 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f8bf24cb421495098feb2b67d83e178, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733788740706 2024-12-09T23:59:06,472 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting d73607d1f6f24d8993615b5f21de18fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733788741981 2024-12-09T23:59:06,473 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c163393f2a14930bbd05c967d1b953c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733788744156 2024-12-09T23:59:06,478 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89b6654328f161896b9d470a5e0e55fb#C#compaction#390 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:59:06,478 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/5e1a5025328d4af8b64fa07c10eae924 is 50, key is test_row_0/C:col10/1733788744161/Put/seqid=0 2024-12-09T23:59:06,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742281_1457 (size=12983) 2024-12-09T23:59:06,513 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/04a787bf57694835a8e977c83b5913f3 2024-12-09T23:59:06,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/e15cdf52c61042228128af1b47e5d7d8 is 50, key is test_row_0/B:col10/1733788744823/Put/seqid=0 2024-12-09T23:59:06,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742282_1458 (size=12301) 2024-12-09T23:59:06,885 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/5e1a5025328d4af8b64fa07c10eae924 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/5e1a5025328d4af8b64fa07c10eae924 2024-12-09T23:59:06,889 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89b6654328f161896b9d470a5e0e55fb/C of 89b6654328f161896b9d470a5e0e55fb into 5e1a5025328d4af8b64fa07c10eae924(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:59:06,889 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:59:06,889 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., storeName=89b6654328f161896b9d470a5e0e55fb/C, priority=13, startTime=1733788746049; duration=0sec 2024-12-09T23:59:06,890 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:59:06,890 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89b6654328f161896b9d470a5e0e55fb:C 2024-12-09T23:59:06,921 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/e15cdf52c61042228128af1b47e5d7d8 2024-12-09T23:59:06,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/256434fa1de944c99c7c92ed5f7ca954 is 50, key is test_row_0/C:col10/1733788744823/Put/seqid=0 2024-12-09T23:59:06,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742283_1459 (size=12301) 2024-12-09T23:59:06,930 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/256434fa1de944c99c7c92ed5f7ca954 2024-12-09T23:59:06,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/04a787bf57694835a8e977c83b5913f3 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/04a787bf57694835a8e977c83b5913f3 2024-12-09T23:59:06,936 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/04a787bf57694835a8e977c83b5913f3, entries=150, sequenceid=317, filesize=12.0 K 2024-12-09T23:59:06,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/e15cdf52c61042228128af1b47e5d7d8 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/e15cdf52c61042228128af1b47e5d7d8 2024-12-09T23:59:06,942 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/e15cdf52c61042228128af1b47e5d7d8, entries=150, sequenceid=317, filesize=12.0 K 2024-12-09T23:59:06,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/256434fa1de944c99c7c92ed5f7ca954 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/256434fa1de944c99c7c92ed5f7ca954 2024-12-09T23:59:06,946 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/256434fa1de944c99c7c92ed5f7ca954, entries=150, sequenceid=317, filesize=12.0 K 2024-12-09T23:59:06,947 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 89b6654328f161896b9d470a5e0e55fb in 842ms, sequenceid=317, compaction requested=false 2024-12-09T23:59:06,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:59:06,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:06,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-09T23:59:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-09T23:59:06,950 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-09T23:59:06,950 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0630 sec 2024-12-09T23:59:06,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 2.0670 sec 2024-12-09T23:59:06,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on 89b6654328f161896b9d470a5e0e55fb 2024-12-09T23:59:06,995 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 89b6654328f161896b9d470a5e0e55fb 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T23:59:06,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=A 2024-12-09T23:59:06,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:06,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=B 2024-12-09T23:59:06,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:06,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=C 2024-12-09T23:59:06,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:07,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/a0dc2d84b0dc444e9e1088f1d0d2bd79 is 50, key is test_row_0/A:col10/1733788746994/Put/seqid=0 2024-12-09T23:59:07,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742284_1460 (size=14741) 2024-12-09T23:59:07,028 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733788747017, max timestamp=1733788747026Num uniques 859; 2024-12-09T23:59:07,028 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733788747017, max timestamp=1733788747026Num uniques 859; 2024-12-09T23:59:07,028 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T23:59:07,028 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T23:59:07,028 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733788747017, max timestamp=1733788747026Num uniques 859; 2024-12-09T23:59:07,028 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T23:59:07,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-09T23:59:07,033 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:59:07,034 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:59:07,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-12-09T23:59:07,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-09T23:59:07,035 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:59:07,036 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:59:07,036 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:59:07,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:07,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788807053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:07,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:07,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:07,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:07,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788807059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:07,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:07,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:07,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:07,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:07,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788807061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:07,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:07,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:07,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:07,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788807062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:07,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:07,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:07,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:07,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-09T23:59:07,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788807172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:07,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:07,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:07,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788807172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:07,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:07,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:07,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:07,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:07,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-09T23:59:07,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:07,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. as already flushing 2024-12-09T23:59:07,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:07,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:07,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:07,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:07,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788807183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:07,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:07,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:07,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:07,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788807183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:07,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:07,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:07,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:07,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:07,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-09T23:59:07,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:07,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. as already flushing 2024-12-09T23:59:07,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:07,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:07,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:07,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:07,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-09T23:59:07,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:07,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788807382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:07,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:07,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:07,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:07,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:07,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788807383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:07,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:07,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:07,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:07,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:07,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788807403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:07,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:07,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:07,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:07,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:07,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788807403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:07,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:07,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:07,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:07,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/a0dc2d84b0dc444e9e1088f1d0d2bd79 2024-12-09T23:59:07,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/08743a5b206b46579db56470a122011b is 50, key is test_row_0/B:col10/1733788746994/Put/seqid=0 2024-12-09T23:59:07,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742285_1461 (size=12301) 2024-12-09T23:59:07,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/08743a5b206b46579db56470a122011b 2024-12-09T23:59:07,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/ed630d8c05f44fa681245b462d0a626e is 50, key is test_row_0/C:col10/1733788746994/Put/seqid=0 2024-12-09T23:59:07,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742286_1462 (size=12301) 2024-12-09T23:59:07,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/ed630d8c05f44fa681245b462d0a626e 2024-12-09T23:59:07,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/a0dc2d84b0dc444e9e1088f1d0d2bd79 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/a0dc2d84b0dc444e9e1088f1d0d2bd79 2024-12-09T23:59:07,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/a0dc2d84b0dc444e9e1088f1d0d2bd79, entries=200, sequenceid=330, filesize=14.4 K 2024-12-09T23:59:07,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/08743a5b206b46579db56470a122011b as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/08743a5b206b46579db56470a122011b 2024-12-09T23:59:07,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/08743a5b206b46579db56470a122011b, entries=150, sequenceid=330, filesize=12.0 K 2024-12-09T23:59:07,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/ed630d8c05f44fa681245b462d0a626e as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/ed630d8c05f44fa681245b462d0a626e 2024-12-09T23:59:07,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/ed630d8c05f44fa681245b462d0a626e, entries=150, sequenceid=330, filesize=12.0 K 2024-12-09T23:59:07,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 89b6654328f161896b9d470a5e0e55fb in 451ms, sequenceid=330, compaction requested=true 2024-12-09T23:59:07,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:59:07,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89b6654328f161896b9d470a5e0e55fb:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:59:07,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:59:07,445 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:59:07,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89b6654328f161896b9d470a5e0e55fb:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:59:07,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:59:07,445 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:59:07,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89b6654328f161896b9d470a5e0e55fb:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:59:07,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:59:07,446 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:59:07,446 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:59:07,446 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): 89b6654328f161896b9d470a5e0e55fb/B is initiating minor compaction (all files) 2024-12-09T23:59:07,446 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): 89b6654328f161896b9d470a5e0e55fb/A is initiating minor compaction (all files) 2024-12-09T23:59:07,446 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89b6654328f161896b9d470a5e0e55fb/B in TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:07,446 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89b6654328f161896b9d470a5e0e55fb/A in TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:07,446 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/e6235512ae4d44f18d7eb244b8394c51, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/e15cdf52c61042228128af1b47e5d7d8, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/08743a5b206b46579db56470a122011b] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp, totalSize=36.7 K 2024-12-09T23:59:07,446 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/aa0d74303db140cfa7230b9d00f99aa7, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/04a787bf57694835a8e977c83b5913f3, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/a0dc2d84b0dc444e9e1088f1d0d2bd79] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp, totalSize=39.1 K 2024-12-09T23:59:07,446 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting e6235512ae4d44f18d7eb244b8394c51, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733788744156 2024-12-09T23:59:07,446 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa0d74303db140cfa7230b9d00f99aa7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733788744156 2024-12-09T23:59:07,447 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting e15cdf52c61042228128af1b47e5d7d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733788744816 2024-12-09T23:59:07,447 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04a787bf57694835a8e977c83b5913f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733788744816 2024-12-09T23:59:07,447 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 08743a5b206b46579db56470a122011b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733788746986 2024-12-09T23:59:07,447 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0dc2d84b0dc444e9e1088f1d0d2bd79, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733788746986 2024-12-09T23:59:07,455 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89b6654328f161896b9d470a5e0e55fb#A#compaction#396 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:59:07,455 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/44903f09c84f41db8775f1b9d3d9e2cb is 50, key is test_row_0/A:col10/1733788746994/Put/seqid=0 2024-12-09T23:59:07,456 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89b6654328f161896b9d470a5e0e55fb#B#compaction#397 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:59:07,457 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/2ec3a0175e594a8ead21352fd1d04a26 is 50, key is test_row_0/B:col10/1733788746994/Put/seqid=0 2024-12-09T23:59:07,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742287_1463 (size=13085) 2024-12-09T23:59:07,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742288_1464 (size=13085) 2024-12-09T23:59:07,467 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/2ec3a0175e594a8ead21352fd1d04a26 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/2ec3a0175e594a8ead21352fd1d04a26 2024-12-09T23:59:07,467 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/44903f09c84f41db8775f1b9d3d9e2cb as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/44903f09c84f41db8775f1b9d3d9e2cb 2024-12-09T23:59:07,473 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89b6654328f161896b9d470a5e0e55fb/A of 89b6654328f161896b9d470a5e0e55fb into 44903f09c84f41db8775f1b9d3d9e2cb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:59:07,473 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89b6654328f161896b9d470a5e0e55fb/B of 89b6654328f161896b9d470a5e0e55fb into 2ec3a0175e594a8ead21352fd1d04a26(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:59:07,473 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:59:07,473 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:59:07,473 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., storeName=89b6654328f161896b9d470a5e0e55fb/A, priority=13, startTime=1733788747445; duration=0sec 2024-12-09T23:59:07,473 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., storeName=89b6654328f161896b9d470a5e0e55fb/B, priority=13, startTime=1733788747445; duration=0sec 2024-12-09T23:59:07,473 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:59:07,473 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89b6654328f161896b9d470a5e0e55fb:A 2024-12-09T23:59:07,473 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:59:07,473 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89b6654328f161896b9d470a5e0e55fb:B 2024-12-09T23:59:07,473 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:59:07,473 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:59:07,473 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): 89b6654328f161896b9d470a5e0e55fb/C is initiating minor compaction (all files) 2024-12-09T23:59:07,473 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89b6654328f161896b9d470a5e0e55fb/C in TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:07,474 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/5e1a5025328d4af8b64fa07c10eae924, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/256434fa1de944c99c7c92ed5f7ca954, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/ed630d8c05f44fa681245b462d0a626e] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp, totalSize=36.7 K 2024-12-09T23:59:07,474 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e1a5025328d4af8b64fa07c10eae924, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733788744156 2024-12-09T23:59:07,474 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 256434fa1de944c99c7c92ed5f7ca954, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733788744816 2024-12-09T23:59:07,474 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed630d8c05f44fa681245b462d0a626e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733788746986 2024-12-09T23:59:07,478 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89b6654328f161896b9d470a5e0e55fb#C#compaction#398 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:59:07,479 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/bc72c20ce5d34056868ea91a3604e206 is 50, key is test_row_0/C:col10/1733788746994/Put/seqid=0 2024-12-09T23:59:07,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742289_1465 (size=13085) 2024-12-09T23:59:07,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-09T23:59:07,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:07,494 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 89b6654328f161896b9d470a5e0e55fb 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-09T23:59:07,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=A 2024-12-09T23:59:07,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:59:07,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=B 2024-12-09T23:59:07,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:59:07,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=C 2024-12-09T23:59:07,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:59:07,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/e5926f48b9594a3fad5610db71f63c6c is 50, key is test_row_0/A:col10/1733788747027/Put/seqid=0 2024-12-09T23:59:07,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742290_1466 (size=12301) 2024-12-09T23:59:07,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-09T23:59:07,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on 89b6654328f161896b9d470a5e0e55fb 2024-12-09T23:59:07,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. as already flushing 2024-12-09T23:59:07,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:07,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788807708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:07,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:07,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:07,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:07,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:07,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788807710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:07,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:07,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:07,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:07,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:07,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788807713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:07,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:07,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:07,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:07,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:07,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788807713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:07,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:07,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:07,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:07,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:07,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788807813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:07,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:07,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:07,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:07,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:07,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788807823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:07,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:07,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:07,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:07,885 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/bc72c20ce5d34056868ea91a3604e206 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/bc72c20ce5d34056868ea91a3604e206 2024-12-09T23:59:07,889 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89b6654328f161896b9d470a5e0e55fb/C of 89b6654328f161896b9d470a5e0e55fb into bc72c20ce5d34056868ea91a3604e206(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:59:07,889 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:59:07,889 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., storeName=89b6654328f161896b9d470a5e0e55fb/C, priority=13, startTime=1733788747445; duration=0sec 2024-12-09T23:59:07,889 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:59:07,889 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89b6654328f161896b9d470a5e0e55fb:C 2024-12-09T23:59:07,900 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/e5926f48b9594a3fad5610db71f63c6c 2024-12-09T23:59:07,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/1bec38202c474dbf85d90cb58977eb49 is 50, key is test_row_0/B:col10/1733788747027/Put/seqid=0 2024-12-09T23:59:07,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742291_1467 (size=12301) 2024-12-09T23:59:08,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:08,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788808022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:08,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:08,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:08,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:08,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:08,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788808033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:08,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:08,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:08,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:08,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-09T23:59:08,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:08,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788808222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:08,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:08,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:08,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:08,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:08,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788808232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:08,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:08,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:08,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:08,292 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T23:59:08,315 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/1bec38202c474dbf85d90cb58977eb49 2024-12-09T23:59:08,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/1c02f24376954b43acd05a7429125d0d is 50, key is test_row_0/C:col10/1733788747027/Put/seqid=0 2024-12-09T23:59:08,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742292_1468 (size=12301) 2024-12-09T23:59:08,325 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/1c02f24376954b43acd05a7429125d0d 2024-12-09T23:59:08,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/e5926f48b9594a3fad5610db71f63c6c as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/e5926f48b9594a3fad5610db71f63c6c 2024-12-09T23:59:08,330 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/e5926f48b9594a3fad5610db71f63c6c, entries=150, sequenceid=367, filesize=12.0 K 2024-12-09T23:59:08,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/1bec38202c474dbf85d90cb58977eb49 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/1bec38202c474dbf85d90cb58977eb49 2024-12-09T23:59:08,333 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/1bec38202c474dbf85d90cb58977eb49, entries=150, sequenceid=367, filesize=12.0 K 2024-12-09T23:59:08,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/1c02f24376954b43acd05a7429125d0d as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/1c02f24376954b43acd05a7429125d0d 2024-12-09T23:59:08,336 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/1c02f24376954b43acd05a7429125d0d, entries=150, sequenceid=367, filesize=12.0 K 2024-12-09T23:59:08,337 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 89b6654328f161896b9d470a5e0e55fb in 844ms, sequenceid=367, compaction requested=false 2024-12-09T23:59:08,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:59:08,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:08,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-12-09T23:59:08,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-12-09T23:59:08,341 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-09T23:59:08,341 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3020 sec 2024-12-09T23:59:08,343 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.3080 sec 2024-12-09T23:59:08,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on 89b6654328f161896b9d470a5e0e55fb 2024-12-09T23:59:08,344 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 89b6654328f161896b9d470a5e0e55fb 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T23:59:08,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=A 2024-12-09T23:59:08,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:08,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=B 2024-12-09T23:59:08,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:08,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=C 2024-12-09T23:59:08,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:08,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/038a4a52f6ed427392b2f2770b5adcc2 is 50, key is test_row_0/A:col10/1733788748344/Put/seqid=0 2024-12-09T23:59:08,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742293_1469 (size=14741) 2024-12-09T23:59:08,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:08,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788808400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:08,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:08,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:08,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:08,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733788748371, max timestamp=1733788748406Num uniques 859; 2024-12-09T23:59:08,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733788748371, max timestamp=1733788748406Num uniques 859; 2024-12-09T23:59:08,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T23:59:08,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T23:59:08,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733788748371, max timestamp=1733788748406Num uniques 859; 2024-12-09T23:59:08,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T23:59:08,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:08,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788808526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:08,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:08,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:08,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:08,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:08,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788808526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:08,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:08,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:08,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:08,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:08,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788808643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:08,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:08,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:08,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:08,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788808643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:08,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:08,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:08,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:08,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:08,752 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/038a4a52f6ed427392b2f2770b5adcc2 2024-12-09T23:59:08,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/4ba8378ff70a4127b486c7b3d12097e9 is 50, key is test_row_0/B:col10/1733788748344/Put/seqid=0 2024-12-09T23:59:08,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742294_1470 (size=12301) 2024-12-09T23:59:08,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:08,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788808852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:08,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:08,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:08,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:08,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:08,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788808863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:08,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:08,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:08,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:08,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51356 deadline: 1733788808973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:08,976 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_1' in 89b6654328f161896b9d470a5e0e55fb of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8250 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:59:08,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:08,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:08,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:09,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/4ba8378ff70a4127b486c7b3d12097e9 2024-12-09T23:59:09,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/f3b43a18b6094922aa7b3aab7727f36f is 50, key is test_row_0/C:col10/1733788748344/Put/seqid=0 2024-12-09T23:59:09,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:09,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788809163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:09,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:09,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:09,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:09,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742295_1471 (size=12301) 2024-12-09T23:59:09,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:09,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788809172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:09,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:09,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:09,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:09,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-09T23:59:09,183 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:59:09,183 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:59:09,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-12-09T23:59:09,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-09T23:59:09,185 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:59:09,185 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:59:09,185 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:59:09,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:09,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1733788809243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:09,249 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_0' in 89b6654328f161896b9d470a5e0e55fb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:59:09,250 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:09,250 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:09,250 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:09,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:09,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733788809252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:09,258 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_1' in 89b6654328f161896b9d470a5e0e55fb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:59:09,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:09,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:09,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-09T23:59:09,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-09T23:59:09,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:09,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. as already flushing 2024-12-09T23:59:09,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:09,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:09,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:09,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-09T23:59:09,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:09,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. as already flushing 2024-12-09T23:59:09,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:09,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:09,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:09,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:09,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-09T23:59:09,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/f3b43a18b6094922aa7b3aab7727f36f 2024-12-09T23:59:09,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/038a4a52f6ed427392b2f2770b5adcc2 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/038a4a52f6ed427392b2f2770b5adcc2 2024-12-09T23:59:09,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/038a4a52f6ed427392b2f2770b5adcc2, entries=200, sequenceid=379, filesize=14.4 K 2024-12-09T23:59:09,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/4ba8378ff70a4127b486c7b3d12097e9 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/4ba8378ff70a4127b486c7b3d12097e9 2024-12-09T23:59:09,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/4ba8378ff70a4127b486c7b3d12097e9, entries=150, sequenceid=379, filesize=12.0 K 2024-12-09T23:59:09,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/f3b43a18b6094922aa7b3aab7727f36f as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/f3b43a18b6094922aa7b3aab7727f36f 2024-12-09T23:59:09,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/f3b43a18b6094922aa7b3aab7727f36f, entries=150, sequenceid=379, filesize=12.0 K 2024-12-09T23:59:09,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 89b6654328f161896b9d470a5e0e55fb in 1241ms, sequenceid=379, compaction requested=true 2024-12-09T23:59:09,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:59:09,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89b6654328f161896b9d470a5e0e55fb:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:59:09,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:59:09,585 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:59:09,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89b6654328f161896b9d470a5e0e55fb:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:59:09,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:59:09,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89b6654328f161896b9d470a5e0e55fb:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:59:09,585 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:59:09,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:59:09,586 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:59:09,586 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): 89b6654328f161896b9d470a5e0e55fb/A is initiating minor compaction (all files) 2024-12-09T23:59:09,586 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89b6654328f161896b9d470a5e0e55fb/A in TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:09,586 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/44903f09c84f41db8775f1b9d3d9e2cb, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/e5926f48b9594a3fad5610db71f63c6c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/038a4a52f6ed427392b2f2770b5adcc2] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp, totalSize=39.2 K 2024-12-09T23:59:09,586 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:59:09,586 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): 89b6654328f161896b9d470a5e0e55fb/B is initiating minor compaction (all files) 2024-12-09T23:59:09,586 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89b6654328f161896b9d470a5e0e55fb/B in TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:09,586 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/2ec3a0175e594a8ead21352fd1d04a26, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/1bec38202c474dbf85d90cb58977eb49, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/4ba8378ff70a4127b486c7b3d12097e9] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp, totalSize=36.8 K 2024-12-09T23:59:09,587 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44903f09c84f41db8775f1b9d3d9e2cb, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733788746986 2024-12-09T23:59:09,587 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ec3a0175e594a8ead21352fd1d04a26, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733788746986 2024-12-09T23:59:09,587 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5926f48b9594a3fad5610db71f63c6c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733788747027 2024-12-09T23:59:09,587 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bec38202c474dbf85d90cb58977eb49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733788747027 2024-12-09T23:59:09,587 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 038a4a52f6ed427392b2f2770b5adcc2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733788747700 2024-12-09T23:59:09,587 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ba8378ff70a4127b486c7b3d12097e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733788747700 2024-12-09T23:59:09,592 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89b6654328f161896b9d470a5e0e55fb#B#compaction#405 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:59:09,592 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/6e0e2b2003fb4b27b40e0f62689485f3 is 50, key is test_row_0/B:col10/1733788748344/Put/seqid=0 2024-12-09T23:59:09,594 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89b6654328f161896b9d470a5e0e55fb#A#compaction#406 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:59:09,594 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/2249898cbb99482aa457851e38ed2d19 is 50, key is test_row_0/A:col10/1733788748344/Put/seqid=0 2024-12-09T23:59:09,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742296_1472 (size=13187) 2024-12-09T23:59:09,602 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/6e0e2b2003fb4b27b40e0f62689485f3 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/6e0e2b2003fb4b27b40e0f62689485f3 2024-12-09T23:59:09,605 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89b6654328f161896b9d470a5e0e55fb/B of 89b6654328f161896b9d470a5e0e55fb into 6e0e2b2003fb4b27b40e0f62689485f3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:59:09,605 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:59:09,605 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., storeName=89b6654328f161896b9d470a5e0e55fb/B, priority=13, startTime=1733788749585; duration=0sec 2024-12-09T23:59:09,605 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:59:09,605 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89b6654328f161896b9d470a5e0e55fb:B 2024-12-09T23:59:09,605 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:59:09,606 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:59:09,606 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): 89b6654328f161896b9d470a5e0e55fb/C is initiating minor compaction (all files) 2024-12-09T23:59:09,606 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89b6654328f161896b9d470a5e0e55fb/C in TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:09,606 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/bc72c20ce5d34056868ea91a3604e206, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/1c02f24376954b43acd05a7429125d0d, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/f3b43a18b6094922aa7b3aab7727f36f] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp, totalSize=36.8 K 2024-12-09T23:59:09,607 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting bc72c20ce5d34056868ea91a3604e206, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733788746986 2024-12-09T23:59:09,607 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c02f24376954b43acd05a7429125d0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733788747027 2024-12-09T23:59:09,607 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting f3b43a18b6094922aa7b3aab7727f36f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733788747700 2024-12-09T23:59:09,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742297_1473 (size=13187) 2024-12-09T23:59:09,614 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/2249898cbb99482aa457851e38ed2d19 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/2249898cbb99482aa457851e38ed2d19 2024-12-09T23:59:09,614 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89b6654328f161896b9d470a5e0e55fb#C#compaction#407 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:59:09,614 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/681735fe6be643a9a7f1b4800f08e929 is 50, key is test_row_0/C:col10/1733788748344/Put/seqid=0 2024-12-09T23:59:09,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742298_1474 (size=13187) 2024-12-09T23:59:09,618 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89b6654328f161896b9d470a5e0e55fb/A of 89b6654328f161896b9d470a5e0e55fb into 2249898cbb99482aa457851e38ed2d19(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:59:09,618 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:59:09,618 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., storeName=89b6654328f161896b9d470a5e0e55fb/A, priority=13, startTime=1733788749585; duration=0sec 2024-12-09T23:59:09,618 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:59:09,618 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89b6654328f161896b9d470a5e0e55fb:A 2024-12-09T23:59:09,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-09T23:59:09,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:09,642 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 89b6654328f161896b9d470a5e0e55fb 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-09T23:59:09,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=A 2024-12-09T23:59:09,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:59:09,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=B 2024-12-09T23:59:09,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:59:09,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=C 2024-12-09T23:59:09,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:59:09,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/e43d5bbe2c524674bb191ef577d0b1ff is 50, key is test_row_0/A:col10/1733788748524/Put/seqid=0 2024-12-09T23:59:09,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742299_1475 (size=12301) 2024-12-09T23:59:09,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on 89b6654328f161896b9d470a5e0e55fb 2024-12-09T23:59:09,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. as already flushing 2024-12-09T23:59:09,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:09,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788809701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:09,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:09,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:09,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:09,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:09,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788809707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:09,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:09,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:09,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:09,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-09T23:59:09,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:09,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788809823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:09,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:09,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:09,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:09,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:09,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788809823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:09,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:09,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:09,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:09,832 INFO [Thread-1802 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1802. 2024-12-09T23:59:09,832 DEBUG [Thread-1802 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:59:09,832 DEBUG [Thread-1802 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:09,832 DEBUG [Thread-1802 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:09,832 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:59:09,832 INFO [Thread-1800 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1800. 2024-12-09T23:59:09,832 DEBUG [Thread-1800 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:59:09,832 DEBUG [Thread-1800 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:09,832 DEBUG [Thread-1800 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:09,832 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:59:09,836 INFO [Thread-1798 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1798. 2024-12-09T23:59:09,836 DEBUG [Thread-1798 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:59:09,836 DEBUG [Thread-1798 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:09,836 DEBUG [Thread-1798 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:09,836 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:59:09,841 INFO [Thread-1806 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1806. 2024-12-09T23:59:09,841 DEBUG [Thread-1806 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:59:09,841 DEBUG [Thread-1806 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:09,841 DEBUG [Thread-1806 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:09,841 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:59:09,841 INFO [Thread-1804 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1804. 2024-12-09T23:59:09,841 DEBUG [Thread-1804 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:59:09,841 DEBUG [Thread-1804 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:09,841 DEBUG [Thread-1804 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:09,841 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:59:10,029 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/681735fe6be643a9a7f1b4800f08e929 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/681735fe6be643a9a7f1b4800f08e929 2024-12-09T23:59:10,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:10,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788810033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:10,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:10,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:10,033 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89b6654328f161896b9d470a5e0e55fb/C of 89b6654328f161896b9d470a5e0e55fb into 681735fe6be643a9a7f1b4800f08e929(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:59:10,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:10,033 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:59:10,033 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., storeName=89b6654328f161896b9d470a5e0e55fb/C, priority=13, startTime=1733788749585; duration=0sec 2024-12-09T23:59:10,034 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:59:10,034 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89b6654328f161896b9d470a5e0e55fb:C 2024-12-09T23:59:10,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:10,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788810043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:10,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:10,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:10,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:10,050 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/e43d5bbe2c524674bb191ef577d0b1ff 2024-12-09T23:59:10,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/4f335c551e014145a47dcce868b21b3b is 50, key is test_row_0/B:col10/1733788748524/Put/seqid=0 2024-12-09T23:59:10,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742300_1476 (size=12301) 2024-12-09T23:59:10,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-09T23:59:10,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:10,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788810343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:10,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:10,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:10,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:10,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:10,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788810353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:10,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:10,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:10,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:10,468 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/4f335c551e014145a47dcce868b21b3b 2024-12-09T23:59:10,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/8bafea405cf74d45b598fe25321cb35f is 50, key is test_row_0/C:col10/1733788748524/Put/seqid=0 2024-12-09T23:59:10,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742301_1477 (size=12301) 2024-12-09T23:59:10,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:10,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733788810854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:10,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:10,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:10,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:10,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:10,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733788810863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:10,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 , the old value is region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:10,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:10,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., hostname=2907d75fbb3e,37627,1733788599837, seqNum=2 because the exception is null or not the one we care about 2024-12-09T23:59:10,887 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/8bafea405cf74d45b598fe25321cb35f 2024-12-09T23:59:10,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/e43d5bbe2c524674bb191ef577d0b1ff as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/e43d5bbe2c524674bb191ef577d0b1ff 2024-12-09T23:59:10,902 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/e43d5bbe2c524674bb191ef577d0b1ff, entries=150, sequenceid=417, filesize=12.0 K 2024-12-09T23:59:10,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/4f335c551e014145a47dcce868b21b3b as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/4f335c551e014145a47dcce868b21b3b 2024-12-09T23:59:10,908 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/4f335c551e014145a47dcce868b21b3b, entries=150, sequenceid=417, filesize=12.0 K 2024-12-09T23:59:10,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/8bafea405cf74d45b598fe25321cb35f as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/8bafea405cf74d45b598fe25321cb35f 2024-12-09T23:59:10,912 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/8bafea405cf74d45b598fe25321cb35f, entries=150, sequenceid=417, filesize=12.0 K 2024-12-09T23:59:10,912 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 89b6654328f161896b9d470a5e0e55fb in 1270ms, sequenceid=417, compaction requested=false 2024-12-09T23:59:10,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:59:10,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:10,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-12-09T23:59:10,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-12-09T23:59:10,915 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-09T23:59:10,916 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7290 sec 2024-12-09T23:59:10,917 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.7330 sec 2024-12-09T23:59:11,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on 89b6654328f161896b9d470a5e0e55fb 2024-12-09T23:59:11,277 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 89b6654328f161896b9d470a5e0e55fb 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T23:59:11,278 INFO [Thread-1789 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1789. 2024-12-09T23:59:11,278 INFO [Thread-1791 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1791. 2024-12-09T23:59:11,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=A 2024-12-09T23:59:11,278 DEBUG [Thread-1789 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:59:11,278 DEBUG [Thread-1791 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:59:11,278 DEBUG [Thread-1789 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:11,278 DEBUG [Thread-1791 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:11,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:11,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=B 2024-12-09T23:59:11,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:11,278 DEBUG [Thread-1791 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:11,278 DEBUG [Thread-1789 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:11,278 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:59:11,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=C 2024-12-09T23:59:11,278 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:59:11,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:11,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/c073250e3f784c74a060ca5a6f5db44b is 50, key is test_row_0/A:col10/1733788751275/Put/seqid=0 2024-12-09T23:59:11,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742302_1478 (size=12301) 2024-12-09T23:59:11,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-09T23:59:11,334 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:59:11,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/c073250e3f784c74a060ca5a6f5db44b 2024-12-09T23:59:11,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/8926868739184ded9829ccc5b7593905 is 50, key is test_row_0/B:col10/1733788751275/Put/seqid=0 2024-12-09T23:59:11,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742303_1479 (size=12301) 2024-12-09T23:59:11,878 INFO [Thread-1793 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1793. 2024-12-09T23:59:11,878 INFO [Thread-1787 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1787. 2024-12-09T23:59:11,878 DEBUG [Thread-1787 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:59:11,878 DEBUG [Thread-1793 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:59:11,878 DEBUG [Thread-1787 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:11,878 DEBUG [Thread-1793 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:11,878 DEBUG [Thread-1787 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:11,878 DEBUG [Thread-1793 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:11,878 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:59:11,878 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:59:12,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/8926868739184ded9829ccc5b7593905 2024-12-09T23:59:12,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/9cbffbb5d5904d9b9b1e38abf4865479 is 50, key is test_row_0/C:col10/1733788751275/Put/seqid=0 2024-12-09T23:59:12,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742304_1480 (size=12301) 2024-12-09T23:59:12,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/9cbffbb5d5904d9b9b1e38abf4865479 2024-12-09T23:59:12,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/c073250e3f784c74a060ca5a6f5db44b as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/c073250e3f784c74a060ca5a6f5db44b 2024-12-09T23:59:12,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/c073250e3f784c74a060ca5a6f5db44b, entries=150, sequenceid=430, filesize=12.0 K 2024-12-09T23:59:12,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/8926868739184ded9829ccc5b7593905 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/8926868739184ded9829ccc5b7593905 2024-12-09T23:59:12,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/8926868739184ded9829ccc5b7593905, entries=150, sequenceid=430, filesize=12.0 K 2024-12-09T23:59:12,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/9cbffbb5d5904d9b9b1e38abf4865479 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/9cbffbb5d5904d9b9b1e38abf4865479 2024-12-09T23:59:12,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/9cbffbb5d5904d9b9b1e38abf4865479, entries=150, sequenceid=430, filesize=12.0 K 2024-12-09T23:59:12,600 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=13.42 KB/13740 for 89b6654328f161896b9d470a5e0e55fb in 1323ms, sequenceid=430, compaction requested=true 2024-12-09T23:59:12,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:59:12,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89b6654328f161896b9d470a5e0e55fb:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:59:12,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:59:12,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89b6654328f161896b9d470a5e0e55fb:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:59:12,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:59:12,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89b6654328f161896b9d470a5e0e55fb:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:59:12,601 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:59:12,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:59:12,601 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:59:12,601 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:59:12,601 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:59:12,602 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): 89b6654328f161896b9d470a5e0e55fb/A is initiating minor compaction (all files) 2024-12-09T23:59:12,602 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): 89b6654328f161896b9d470a5e0e55fb/B is initiating minor compaction (all files) 2024-12-09T23:59:12,602 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89b6654328f161896b9d470a5e0e55fb/A in TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:12,602 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89b6654328f161896b9d470a5e0e55fb/B in TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:12,602 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/2249898cbb99482aa457851e38ed2d19, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/e43d5bbe2c524674bb191ef577d0b1ff, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/c073250e3f784c74a060ca5a6f5db44b] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp, totalSize=36.9 K 2024-12-09T23:59:12,602 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/6e0e2b2003fb4b27b40e0f62689485f3, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/4f335c551e014145a47dcce868b21b3b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/8926868739184ded9829ccc5b7593905] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp, totalSize=36.9 K 2024-12-09T23:59:12,602 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2249898cbb99482aa457851e38ed2d19, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733788747700 2024-12-09T23:59:12,602 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e0e2b2003fb4b27b40e0f62689485f3, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733788747700 2024-12-09T23:59:12,602 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting e43d5bbe2c524674bb191ef577d0b1ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733788748474 2024-12-09T23:59:12,602 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f335c551e014145a47dcce868b21b3b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733788748474 2024-12-09T23:59:12,603 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting c073250e3f784c74a060ca5a6f5db44b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1733788749706 2024-12-09T23:59:12,603 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 8926868739184ded9829ccc5b7593905, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1733788749706 2024-12-09T23:59:12,609 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89b6654328f161896b9d470a5e0e55fb#B#compaction#414 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:59:12,609 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89b6654328f161896b9d470a5e0e55fb#A#compaction#415 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:59:12,609 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/265e05c37b6c4323b90f850d599fbb9a is 50, key is test_row_0/A:col10/1733788751275/Put/seqid=0 2024-12-09T23:59:12,609 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/e16f114775c749bb86d9aaf19c02d69d is 50, key is test_row_0/B:col10/1733788751275/Put/seqid=0 2024-12-09T23:59:12,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742305_1481 (size=13289) 2024-12-09T23:59:12,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742306_1482 (size=13289) 2024-12-09T23:59:13,022 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/e16f114775c749bb86d9aaf19c02d69d as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/e16f114775c749bb86d9aaf19c02d69d 2024-12-09T23:59:13,022 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/265e05c37b6c4323b90f850d599fbb9a as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/265e05c37b6c4323b90f850d599fbb9a 2024-12-09T23:59:13,027 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89b6654328f161896b9d470a5e0e55fb/B of 89b6654328f161896b9d470a5e0e55fb into e16f114775c749bb86d9aaf19c02d69d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:59:13,027 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89b6654328f161896b9d470a5e0e55fb/A of 89b6654328f161896b9d470a5e0e55fb into 265e05c37b6c4323b90f850d599fbb9a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:59:13,027 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:59:13,027 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:59:13,027 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., storeName=89b6654328f161896b9d470a5e0e55fb/A, priority=13, startTime=1733788752600; duration=0sec 2024-12-09T23:59:13,027 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., storeName=89b6654328f161896b9d470a5e0e55fb/B, priority=13, startTime=1733788752600; duration=0sec 2024-12-09T23:59:13,027 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:59:13,027 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:59:13,027 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89b6654328f161896b9d470a5e0e55fb:B 2024-12-09T23:59:13,027 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89b6654328f161896b9d470a5e0e55fb:A 2024-12-09T23:59:13,027 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:59:13,028 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:59:13,028 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): 89b6654328f161896b9d470a5e0e55fb/C is initiating minor compaction (all files) 2024-12-09T23:59:13,028 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89b6654328f161896b9d470a5e0e55fb/C in TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:13,028 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/681735fe6be643a9a7f1b4800f08e929, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/8bafea405cf74d45b598fe25321cb35f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/9cbffbb5d5904d9b9b1e38abf4865479] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp, totalSize=36.9 K 2024-12-09T23:59:13,028 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 681735fe6be643a9a7f1b4800f08e929, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733788747700 2024-12-09T23:59:13,029 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bafea405cf74d45b598fe25321cb35f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733788748474 2024-12-09T23:59:13,029 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cbffbb5d5904d9b9b1e38abf4865479, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1733788749706 2024-12-09T23:59:13,036 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89b6654328f161896b9d470a5e0e55fb#C#compaction#416 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:59:13,037 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/f241ae2845e541cbaba1a7153e2f817d is 50, key is test_row_0/C:col10/1733788751275/Put/seqid=0 2024-12-09T23:59:13,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742307_1483 (size=13289) 2024-12-09T23:59:13,452 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/f241ae2845e541cbaba1a7153e2f817d as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/f241ae2845e541cbaba1a7153e2f817d 2024-12-09T23:59:13,458 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89b6654328f161896b9d470a5e0e55fb/C of 89b6654328f161896b9d470a5e0e55fb into f241ae2845e541cbaba1a7153e2f817d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:59:13,458 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89b6654328f161896b9d470a5e0e55fb: 2024-12-09T23:59:13,458 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., storeName=89b6654328f161896b9d470a5e0e55fb/C, priority=13, startTime=1733788752600; duration=0sec 2024-12-09T23:59:13,458 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:59:13,458 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89b6654328f161896b9d470a5e0e55fb:C 2024-12-09T23:59:19,064 INFO [Thread-1795 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1795. 2024-12-09T23:59:19,064 DEBUG [Thread-1795 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:59:19,064 DEBUG [Thread-1795 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:19,064 DEBUG [Thread-1795 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:19,064 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:59:19,064 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T23:59:19,064 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-12-09T23:59:19,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-12-09T23:59:19,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-12-09T23:59:19,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-12-09T23:59:19,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-12-09T23:59:19,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T23:59:19,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T23:59:19,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1818 2024-12-09T23:59:19,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5450 rows 2024-12-09T23:59:19,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1798 2024-12-09T23:59:19,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5393 rows 2024-12-09T23:59:19,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1813 2024-12-09T23:59:19,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5435 rows 2024-12-09T23:59:19,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1800 2024-12-09T23:59:19,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5396 rows 2024-12-09T23:59:19,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1797 2024-12-09T23:59:19,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5390 rows 2024-12-09T23:59:19,065 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T23:59:19,065 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T23:59:19,065 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:19,065 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:19,065 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:59:19,066 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-09T23:59:19,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T23:59:19,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-09T23:59:19,069 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733788759069"}]},"ts":"1733788759069"} 2024-12-09T23:59:19,070 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T23:59:19,070 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T23:59:19,071 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T23:59:19,072 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=89b6654328f161896b9d470a5e0e55fb, UNASSIGN}] 2024-12-09T23:59:19,073 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=89b6654328f161896b9d470a5e0e55fb, UNASSIGN 2024-12-09T23:59:19,074 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=89b6654328f161896b9d470a5e0e55fb, regionState=CLOSING, regionLocation=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:19,075 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=89b6654328f161896b9d470a5e0e55fb, UNASSIGN because future has completed 2024-12-09T23:59:19,076 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T23:59:19,076 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837}] 2024-12-09T23:59:19,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-09T23:59:19,228 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close 89b6654328f161896b9d470a5e0e55fb 2024-12-09T23:59:19,228 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T23:59:19,228 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing 89b6654328f161896b9d470a5e0e55fb, disabling compactions & flushes 2024-12-09T23:59:19,228 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:19,228 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:19,228 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. after waiting 0 ms 2024-12-09T23:59:19,228 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:19,228 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(2902): Flushing 89b6654328f161896b9d470a5e0e55fb 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-09T23:59:19,228 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=A 2024-12-09T23:59:19,229 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:19,229 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=B 2024-12-09T23:59:19,229 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:19,229 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89b6654328f161896b9d470a5e0e55fb, store=C 2024-12-09T23:59:19,229 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:19,232 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/6ffe7f9858eb4ab28d65ccca6ce33bd1 is 50, key is test_row_0/A:col10/1733788751875/Put/seqid=0 2024-12-09T23:59:19,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742308_1484 (size=12301) 2024-12-09T23:59:19,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-09T23:59:19,637 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/6ffe7f9858eb4ab28d65ccca6ce33bd1 2024-12-09T23:59:19,651 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/4e864b42b8cf4c6d870b5fb9e1ebea52 is 50, key is test_row_0/B:col10/1733788751875/Put/seqid=0 2024-12-09T23:59:19,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742309_1485 (size=12301) 2024-12-09T23:59:19,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-09T23:59:20,058 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/4e864b42b8cf4c6d870b5fb9e1ebea52 2024-12-09T23:59:20,072 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/bc824217274943f68cf126e1e32b4839 is 50, key is test_row_0/C:col10/1733788751875/Put/seqid=0 2024-12-09T23:59:20,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742310_1486 (size=12301) 2024-12-09T23:59:20,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-09T23:59:20,478 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/bc824217274943f68cf126e1e32b4839 2024-12-09T23:59:20,488 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/A/6ffe7f9858eb4ab28d65ccca6ce33bd1 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/6ffe7f9858eb4ab28d65ccca6ce33bd1 2024-12-09T23:59:20,494 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/6ffe7f9858eb4ab28d65ccca6ce33bd1, entries=150, sequenceid=439, filesize=12.0 K 2024-12-09T23:59:20,495 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/B/4e864b42b8cf4c6d870b5fb9e1ebea52 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/4e864b42b8cf4c6d870b5fb9e1ebea52 2024-12-09T23:59:20,499 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/4e864b42b8cf4c6d870b5fb9e1ebea52, entries=150, sequenceid=439, filesize=12.0 K 2024-12-09T23:59:20,500 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/.tmp/C/bc824217274943f68cf126e1e32b4839 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/bc824217274943f68cf126e1e32b4839 2024-12-09T23:59:20,504 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/bc824217274943f68cf126e1e32b4839, entries=150, sequenceid=439, filesize=12.0 K 2024-12-09T23:59:20,505 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 89b6654328f161896b9d470a5e0e55fb in 1277ms, sequenceid=439, compaction requested=false 2024-12-09T23:59:20,505 DEBUG [StoreCloser-TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/c7e729ff16924b8990a7ca066e7b2baf, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/6ef155a2634140d3b3bdbd9c55d5caea, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/fd7a6afe120341c9800a5f9a7eadc55f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/c97c63a9176a4f8cbad55f3bae789f5f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/b0d0d55c26f147b697d2f16bcfbc03ea, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/c5f24ed5ffe1458d975964891cc2ec63, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/c14e98b884d74b20aca964a1b1cd07e7, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/8b5ceea34c55443cb7f00f5c65a8bfec, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/743c280d45aa4d038ad8bdf59e2eec47, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/2c59f1d7c7854531bef87a2130fe21c6, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/cf9655e8201a4c9f835e5ed56193bf74, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/84e80156e38f4cb9898eb2ba92d4da4c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/588467211f2e4dd7bde979687a3cec99, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/c3f3c392115a41bbbf41587af5c69abd, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/34c00421a06b476480d19f70ddcb377e, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/fcbaf39f397248bea0549ba11ffce3ff, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/634507fbbac34b9ea23fba6fd82e037b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/627ddb1c6671457da4310d1b7566a61b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/efac0c882fec419cbf29f430b57418dd, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/d4e24f32cac542e5979c7d781269ab19, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/aa0d74303db140cfa7230b9d00f99aa7, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/04a787bf57694835a8e977c83b5913f3, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/a0dc2d84b0dc444e9e1088f1d0d2bd79, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/44903f09c84f41db8775f1b9d3d9e2cb, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/e5926f48b9594a3fad5610db71f63c6c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/038a4a52f6ed427392b2f2770b5adcc2, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/2249898cbb99482aa457851e38ed2d19, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/e43d5bbe2c524674bb191ef577d0b1ff, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/c073250e3f784c74a060ca5a6f5db44b] to archive 2024-12-09T23:59:20,506 DEBUG [StoreCloser-TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-09T23:59:20,509 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/6ef155a2634140d3b3bdbd9c55d5caea to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/6ef155a2634140d3b3bdbd9c55d5caea 2024-12-09T23:59:20,509 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/c7e729ff16924b8990a7ca066e7b2baf to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/c7e729ff16924b8990a7ca066e7b2baf 2024-12-09T23:59:20,510 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/c97c63a9176a4f8cbad55f3bae789f5f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/c97c63a9176a4f8cbad55f3bae789f5f 2024-12-09T23:59:20,510 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/b0d0d55c26f147b697d2f16bcfbc03ea to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/b0d0d55c26f147b697d2f16bcfbc03ea 2024-12-09T23:59:20,510 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/fd7a6afe120341c9800a5f9a7eadc55f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/fd7a6afe120341c9800a5f9a7eadc55f 2024-12-09T23:59:20,510 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/c14e98b884d74b20aca964a1b1cd07e7 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/c14e98b884d74b20aca964a1b1cd07e7 2024-12-09T23:59:20,510 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/c5f24ed5ffe1458d975964891cc2ec63 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/c5f24ed5ffe1458d975964891cc2ec63 2024-12-09T23:59:20,510 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/8b5ceea34c55443cb7f00f5c65a8bfec to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/8b5ceea34c55443cb7f00f5c65a8bfec 2024-12-09T23:59:20,510 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/743c280d45aa4d038ad8bdf59e2eec47 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/743c280d45aa4d038ad8bdf59e2eec47 2024-12-09T23:59:20,511 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/2c59f1d7c7854531bef87a2130fe21c6 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/2c59f1d7c7854531bef87a2130fe21c6 2024-12-09T23:59:20,511 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/588467211f2e4dd7bde979687a3cec99 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/588467211f2e4dd7bde979687a3cec99 2024-12-09T23:59:20,512 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/84e80156e38f4cb9898eb2ba92d4da4c to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/84e80156e38f4cb9898eb2ba92d4da4c 2024-12-09T23:59:20,512 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/cf9655e8201a4c9f835e5ed56193bf74 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/cf9655e8201a4c9f835e5ed56193bf74 2024-12-09T23:59:20,512 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/c3f3c392115a41bbbf41587af5c69abd to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/c3f3c392115a41bbbf41587af5c69abd 2024-12-09T23:59:20,512 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/fcbaf39f397248bea0549ba11ffce3ff to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/fcbaf39f397248bea0549ba11ffce3ff 2024-12-09T23:59:20,512 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/34c00421a06b476480d19f70ddcb377e to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/34c00421a06b476480d19f70ddcb377e 2024-12-09T23:59:20,512 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/627ddb1c6671457da4310d1b7566a61b to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/627ddb1c6671457da4310d1b7566a61b 2024-12-09T23:59:20,513 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/634507fbbac34b9ea23fba6fd82e037b to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/634507fbbac34b9ea23fba6fd82e037b 2024-12-09T23:59:20,513 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/aa0d74303db140cfa7230b9d00f99aa7 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/aa0d74303db140cfa7230b9d00f99aa7 2024-12-09T23:59:20,513 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/efac0c882fec419cbf29f430b57418dd to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/efac0c882fec419cbf29f430b57418dd 2024-12-09T23:59:20,513 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/04a787bf57694835a8e977c83b5913f3 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/04a787bf57694835a8e977c83b5913f3 2024-12-09T23:59:20,514 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/d4e24f32cac542e5979c7d781269ab19 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/d4e24f32cac542e5979c7d781269ab19 2024-12-09T23:59:20,514 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/a0dc2d84b0dc444e9e1088f1d0d2bd79 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/a0dc2d84b0dc444e9e1088f1d0d2bd79 2024-12-09T23:59:20,514 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/44903f09c84f41db8775f1b9d3d9e2cb to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/44903f09c84f41db8775f1b9d3d9e2cb 2024-12-09T23:59:20,514 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/038a4a52f6ed427392b2f2770b5adcc2 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/038a4a52f6ed427392b2f2770b5adcc2 2024-12-09T23:59:20,514 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/e5926f48b9594a3fad5610db71f63c6c to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/e5926f48b9594a3fad5610db71f63c6c 2024-12-09T23:59:20,514 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/2249898cbb99482aa457851e38ed2d19 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/2249898cbb99482aa457851e38ed2d19 2024-12-09T23:59:20,514 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/c073250e3f784c74a060ca5a6f5db44b to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/c073250e3f784c74a060ca5a6f5db44b 2024-12-09T23:59:20,514 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/e43d5bbe2c524674bb191ef577d0b1ff to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/e43d5bbe2c524674bb191ef577d0b1ff 2024-12-09T23:59:20,516 DEBUG [StoreCloser-TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/c896c0b6a64f4d2db280060e423978b9, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/bd09b9f94c89468a86af9dd394031de3, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/02a8981061ed4334bb99c04a2af8a326, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/78b13749d4614df1876158633c245de1, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/46e4fca786fd4fad8211f25602e3835b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/f8e7445498e848029d39914c06145d3a, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/b1f2b5c8738c4006ad9a459a0e5091d8, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/e2f2225719004f9d9f3e8bf1367a69c9, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/e153ac1233d547459831267a8c171520, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/aeae5294efef4d1e81efe0d6a2e71c65, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/ba422969416b4f919090eef2270d0e1e, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/66a4e06e75e9408788b308dc6168e4e3, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/1d2760caf9d7460d92338d4eea3d2530, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/8deef2bf19d5429684e3da3eae0469bb, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/956632b4262643fc8723ec23da37a70f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/886f73fae48e414a84623724354865f8, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/d7f0f470a0264e39b9fd206fac059f49, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/bda901fa1be24a3c9a12474160da9832, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/0a660c8ce53240beb4679f8c2afe2e9d, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/e6235512ae4d44f18d7eb244b8394c51, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/1abecd15dc934d17b261a60bb6ceddce, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/e15cdf52c61042228128af1b47e5d7d8, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/2ec3a0175e594a8ead21352fd1d04a26, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/08743a5b206b46579db56470a122011b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/1bec38202c474dbf85d90cb58977eb49, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/6e0e2b2003fb4b27b40e0f62689485f3, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/4ba8378ff70a4127b486c7b3d12097e9, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/4f335c551e014145a47dcce868b21b3b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/8926868739184ded9829ccc5b7593905] to archive 2024-12-09T23:59:20,516 DEBUG [StoreCloser-TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-09T23:59:20,518 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/02a8981061ed4334bb99c04a2af8a326 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/02a8981061ed4334bb99c04a2af8a326 2024-12-09T23:59:20,518 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/c896c0b6a64f4d2db280060e423978b9 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/c896c0b6a64f4d2db280060e423978b9 2024-12-09T23:59:20,518 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/46e4fca786fd4fad8211f25602e3835b to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/46e4fca786fd4fad8211f25602e3835b 2024-12-09T23:59:20,518 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/b1f2b5c8738c4006ad9a459a0e5091d8 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/b1f2b5c8738c4006ad9a459a0e5091d8 2024-12-09T23:59:20,518 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/f8e7445498e848029d39914c06145d3a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/f8e7445498e848029d39914c06145d3a 2024-12-09T23:59:20,518 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/78b13749d4614df1876158633c245de1 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/78b13749d4614df1876158633c245de1 2024-12-09T23:59:20,518 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/e2f2225719004f9d9f3e8bf1367a69c9 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/e2f2225719004f9d9f3e8bf1367a69c9 2024-12-09T23:59:20,519 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/bd09b9f94c89468a86af9dd394031de3 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/bd09b9f94c89468a86af9dd394031de3 2024-12-09T23:59:20,519 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/1d2760caf9d7460d92338d4eea3d2530 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/1d2760caf9d7460d92338d4eea3d2530 2024-12-09T23:59:20,523 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/e153ac1233d547459831267a8c171520 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/e153ac1233d547459831267a8c171520 2024-12-09T23:59:20,523 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/956632b4262643fc8723ec23da37a70f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/956632b4262643fc8723ec23da37a70f 2024-12-09T23:59:20,523 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/ba422969416b4f919090eef2270d0e1e to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/ba422969416b4f919090eef2270d0e1e 2024-12-09T23:59:20,523 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/aeae5294efef4d1e81efe0d6a2e71c65 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/aeae5294efef4d1e81efe0d6a2e71c65 2024-12-09T23:59:20,523 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/66a4e06e75e9408788b308dc6168e4e3 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/66a4e06e75e9408788b308dc6168e4e3 2024-12-09T23:59:20,523 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/8deef2bf19d5429684e3da3eae0469bb to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/8deef2bf19d5429684e3da3eae0469bb 2024-12-09T23:59:20,523 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/886f73fae48e414a84623724354865f8 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/886f73fae48e414a84623724354865f8 2024-12-09T23:59:20,524 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/d7f0f470a0264e39b9fd206fac059f49 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/d7f0f470a0264e39b9fd206fac059f49 2024-12-09T23:59:20,524 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/e6235512ae4d44f18d7eb244b8394c51 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/e6235512ae4d44f18d7eb244b8394c51 2024-12-09T23:59:20,524 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/1abecd15dc934d17b261a60bb6ceddce to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/1abecd15dc934d17b261a60bb6ceddce 2024-12-09T23:59:20,524 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/bda901fa1be24a3c9a12474160da9832 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/bda901fa1be24a3c9a12474160da9832 2024-12-09T23:59:20,524 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/0a660c8ce53240beb4679f8c2afe2e9d to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/0a660c8ce53240beb4679f8c2afe2e9d 2024-12-09T23:59:20,524 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/2ec3a0175e594a8ead21352fd1d04a26 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/2ec3a0175e594a8ead21352fd1d04a26 2024-12-09T23:59:20,524 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/08743a5b206b46579db56470a122011b to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/08743a5b206b46579db56470a122011b 2024-12-09T23:59:20,525 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/e15cdf52c61042228128af1b47e5d7d8 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/e15cdf52c61042228128af1b47e5d7d8 2024-12-09T23:59:20,525 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/1bec38202c474dbf85d90cb58977eb49 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/1bec38202c474dbf85d90cb58977eb49 2024-12-09T23:59:20,525 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/6e0e2b2003fb4b27b40e0f62689485f3 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/6e0e2b2003fb4b27b40e0f62689485f3 2024-12-09T23:59:20,525 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/4ba8378ff70a4127b486c7b3d12097e9 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/4ba8378ff70a4127b486c7b3d12097e9 2024-12-09T23:59:20,525 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/8926868739184ded9829ccc5b7593905 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/8926868739184ded9829ccc5b7593905 2024-12-09T23:59:20,525 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/4f335c551e014145a47dcce868b21b3b to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/4f335c551e014145a47dcce868b21b3b 2024-12-09T23:59:20,526 DEBUG [StoreCloser-TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/72349c755c0c497095dd54a8bf095ab5, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/5d9a2ff3e62348fb8a854d62932d63bb, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/01a2f54f7d784faa8cf5505191b5ae27, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/3d49a9ebbcf24c128b41bb57e881a93b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/fa2d72bfc22b4558baba577797bfba71, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/9930a9d5f49e4747842b5337a28cda8b, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/7941b97b10b941e3ad29b4681bbcdd1a, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/fb215f3cb33f4119a25d2f1a911156e8, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/991021ce8d35487daf105b64c7dab6ef, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/82094266df7c48769b01c4a7770742a3, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/6a6ecfa610b341cfae8f7f0ac156961d, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/447ff07340ac4ae9b5a68cd212ed05a9, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/da7863c110eb42c49c028c4bb81a97a6, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/e60e19d9adb845429abab94f0beffca4, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/caa2d7e7b0a24ec08d14902be4ff9500, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/a1521de89e0b44ffbe6860cf8a3526b7, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/5f8bf24cb421495098feb2b67d83e178, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/ca1a679f3f6d428d9f6cf0a160060748, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/d73607d1f6f24d8993615b5f21de18fe, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/5e1a5025328d4af8b64fa07c10eae924, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/5c163393f2a14930bbd05c967d1b953c, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/256434fa1de944c99c7c92ed5f7ca954, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/bc72c20ce5d34056868ea91a3604e206, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/ed630d8c05f44fa681245b462d0a626e, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/1c02f24376954b43acd05a7429125d0d, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/681735fe6be643a9a7f1b4800f08e929, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/f3b43a18b6094922aa7b3aab7727f36f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/8bafea405cf74d45b598fe25321cb35f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/9cbffbb5d5904d9b9b1e38abf4865479] to archive 2024-12-09T23:59:20,527 DEBUG [StoreCloser-TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-09T23:59:20,528 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/3d49a9ebbcf24c128b41bb57e881a93b to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/3d49a9ebbcf24c128b41bb57e881a93b 2024-12-09T23:59:20,529 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/01a2f54f7d784faa8cf5505191b5ae27 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/01a2f54f7d784faa8cf5505191b5ae27 2024-12-09T23:59:20,529 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/72349c755c0c497095dd54a8bf095ab5 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/72349c755c0c497095dd54a8bf095ab5 2024-12-09T23:59:20,529 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/fa2d72bfc22b4558baba577797bfba71 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/fa2d72bfc22b4558baba577797bfba71 2024-12-09T23:59:20,529 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/9930a9d5f49e4747842b5337a28cda8b to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/9930a9d5f49e4747842b5337a28cda8b 2024-12-09T23:59:20,529 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/fb215f3cb33f4119a25d2f1a911156e8 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/fb215f3cb33f4119a25d2f1a911156e8 2024-12-09T23:59:20,529 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/5d9a2ff3e62348fb8a854d62932d63bb to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/5d9a2ff3e62348fb8a854d62932d63bb 2024-12-09T23:59:20,529 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/7941b97b10b941e3ad29b4681bbcdd1a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/7941b97b10b941e3ad29b4681bbcdd1a 2024-12-09T23:59:20,530 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/991021ce8d35487daf105b64c7dab6ef to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/991021ce8d35487daf105b64c7dab6ef 2024-12-09T23:59:20,530 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/82094266df7c48769b01c4a7770742a3 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/82094266df7c48769b01c4a7770742a3 2024-12-09T23:59:20,530 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/e60e19d9adb845429abab94f0beffca4 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/e60e19d9adb845429abab94f0beffca4 2024-12-09T23:59:20,530 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/6a6ecfa610b341cfae8f7f0ac156961d to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/6a6ecfa610b341cfae8f7f0ac156961d 2024-12-09T23:59:20,530 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/caa2d7e7b0a24ec08d14902be4ff9500 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/caa2d7e7b0a24ec08d14902be4ff9500 2024-12-09T23:59:20,530 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/447ff07340ac4ae9b5a68cd212ed05a9 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/447ff07340ac4ae9b5a68cd212ed05a9 2024-12-09T23:59:20,531 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/a1521de89e0b44ffbe6860cf8a3526b7 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/a1521de89e0b44ffbe6860cf8a3526b7 2024-12-09T23:59:20,531 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/da7863c110eb42c49c028c4bb81a97a6 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/da7863c110eb42c49c028c4bb81a97a6 2024-12-09T23:59:20,531 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/5f8bf24cb421495098feb2b67d83e178 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/5f8bf24cb421495098feb2b67d83e178 2024-12-09T23:59:20,532 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/5e1a5025328d4af8b64fa07c10eae924 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/5e1a5025328d4af8b64fa07c10eae924 2024-12-09T23:59:20,532 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/d73607d1f6f24d8993615b5f21de18fe to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/d73607d1f6f24d8993615b5f21de18fe 2024-12-09T23:59:20,532 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/ca1a679f3f6d428d9f6cf0a160060748 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/ca1a679f3f6d428d9f6cf0a160060748 2024-12-09T23:59:20,532 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/5c163393f2a14930bbd05c967d1b953c to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/5c163393f2a14930bbd05c967d1b953c 2024-12-09T23:59:20,532 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/bc72c20ce5d34056868ea91a3604e206 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/bc72c20ce5d34056868ea91a3604e206 2024-12-09T23:59:20,532 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/256434fa1de944c99c7c92ed5f7ca954 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/256434fa1de944c99c7c92ed5f7ca954 2024-12-09T23:59:20,532 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/ed630d8c05f44fa681245b462d0a626e to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/ed630d8c05f44fa681245b462d0a626e 2024-12-09T23:59:20,533 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/1c02f24376954b43acd05a7429125d0d to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/1c02f24376954b43acd05a7429125d0d 2024-12-09T23:59:20,533 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/f3b43a18b6094922aa7b3aab7727f36f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/f3b43a18b6094922aa7b3aab7727f36f 2024-12-09T23:59:20,533 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/681735fe6be643a9a7f1b4800f08e929 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/681735fe6be643a9a7f1b4800f08e929 2024-12-09T23:59:20,533 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/8bafea405cf74d45b598fe25321cb35f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/8bafea405cf74d45b598fe25321cb35f 2024-12-09T23:59:20,533 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/9cbffbb5d5904d9b9b1e38abf4865479 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/9cbffbb5d5904d9b9b1e38abf4865479 2024-12-09T23:59:20,536 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/recovered.edits/442.seqid, newMaxSeqId=442, maxSeqId=1 2024-12-09T23:59:20,537 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. 2024-12-09T23:59:20,537 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for 89b6654328f161896b9d470a5e0e55fb: Waiting for close lock at 1733788759228Running coprocessor pre-close hooks at 1733788759228Disabling compacts and flushes for region at 1733788759228Disabling writes for close at 1733788759228Obtaining lock to block concurrent updates at 1733788759228Preparing flush snapshotting stores in 89b6654328f161896b9d470a5e0e55fb at 1733788759228Finished memstore snapshotting TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733788759229 (+1 ms)Flushing stores of TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb. at 1733788759229Flushing 89b6654328f161896b9d470a5e0e55fb/A: creating writer at 1733788759229Flushing 89b6654328f161896b9d470a5e0e55fb/A: appending metadata at 1733788759232 (+3 ms)Flushing 89b6654328f161896b9d470a5e0e55fb/A: closing flushed file at 1733788759232Flushing 89b6654328f161896b9d470a5e0e55fb/B: creating writer at 1733788759646 (+414 ms)Flushing 89b6654328f161896b9d470a5e0e55fb/B: appending metadata at 1733788759650 (+4 ms)Flushing 89b6654328f161896b9d470a5e0e55fb/B: closing flushed file at 1733788759650Flushing 89b6654328f161896b9d470a5e0e55fb/C: creating writer at 1733788760068 (+418 ms)Flushing 89b6654328f161896b9d470a5e0e55fb/C: appending metadata at 1733788760072 (+4 ms)Flushing 89b6654328f161896b9d470a5e0e55fb/C: closing flushed file at 1733788760072Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7efa74cf: reopening flushed file at 1733788760486 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3bf49c48: reopening flushed file at 1733788760494 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39ad87f3: reopening flushed file at 1733788760499 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 89b6654328f161896b9d470a5e0e55fb in 1277ms, sequenceid=439, compaction requested=false at 1733788760505 (+6 ms)Writing region close event to WAL at 1733788760534 (+29 ms)Running coprocessor post-close hooks at 1733788760537 (+3 ms)Closed at 1733788760537 2024-12-09T23:59:20,539 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed 89b6654328f161896b9d470a5e0e55fb 2024-12-09T23:59:20,539 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=89b6654328f161896b9d470a5e0e55fb, regionState=CLOSED 2024-12-09T23:59:20,540 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 because future has completed 2024-12-09T23:59:20,542 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-09T23:59:20,542 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure 89b6654328f161896b9d470a5e0e55fb, server=2907d75fbb3e,37627,1733788599837 in 1.4650 sec 2024-12-09T23:59:20,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-12-09T23:59:20,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=89b6654328f161896b9d470a5e0e55fb, UNASSIGN in 1.4700 sec 2024-12-09T23:59:20,544 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-09T23:59:20,545 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4720 sec 2024-12-09T23:59:20,545 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733788760545"}]},"ts":"1733788760545"} 2024-12-09T23:59:20,546 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T23:59:20,546 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T23:59:20,548 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4810 sec 2024-12-09T23:59:21,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-09T23:59:21,214 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:59:21,215 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-09T23:59:21,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T23:59:21,219 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T23:59:21,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-09T23:59:21,221 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=141, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T23:59:21,226 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb 2024-12-09T23:59:21,229 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A, FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B, FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C, FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/recovered.edits] 2024-12-09T23:59:21,233 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/6ffe7f9858eb4ab28d65ccca6ce33bd1 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/6ffe7f9858eb4ab28d65ccca6ce33bd1 2024-12-09T23:59:21,233 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/265e05c37b6c4323b90f850d599fbb9a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/A/265e05c37b6c4323b90f850d599fbb9a 2024-12-09T23:59:21,235 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/4e864b42b8cf4c6d870b5fb9e1ebea52 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/4e864b42b8cf4c6d870b5fb9e1ebea52 2024-12-09T23:59:21,236 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/e16f114775c749bb86d9aaf19c02d69d to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/B/e16f114775c749bb86d9aaf19c02d69d 2024-12-09T23:59:21,238 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/bc824217274943f68cf126e1e32b4839 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/bc824217274943f68cf126e1e32b4839 2024-12-09T23:59:21,238 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/f241ae2845e541cbaba1a7153e2f817d to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/C/f241ae2845e541cbaba1a7153e2f817d 2024-12-09T23:59:21,241 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/recovered.edits/442.seqid to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb/recovered.edits/442.seqid 2024-12-09T23:59:21,242 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/89b6654328f161896b9d470a5e0e55fb 2024-12-09T23:59:21,242 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T23:59:21,245 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=141, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T23:59:21,248 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T23:59:21,251 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T23:59:21,253 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=141, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T23:59:21,253 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-09T23:59:21,253 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733788761253"}]},"ts":"9223372036854775807"} 2024-12-09T23:59:21,256 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T23:59:21,256 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 89b6654328f161896b9d470a5e0e55fb, NAME => 'TestAcidGuarantees,,1733788727609.89b6654328f161896b9d470a5e0e55fb.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T23:59:21,257 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T23:59:21,257 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733788761257"}]},"ts":"9223372036854775807"} 2024-12-09T23:59:21,259 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T23:59:21,259 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=141, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T23:59:21,260 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 44 msec 2024-12-09T23:59:21,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-09T23:59:21,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T23:59:21,324 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:59:21,335 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=254 (was 255), OpenFileDescriptor=451 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=256 (was 309), ProcessCount=11 (was 11), AvailableMemoryMB=3993 (was 3975) - AvailableMemoryMB LEAK? - 2024-12-09T23:59:21,346 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=254, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=256, ProcessCount=11, AvailableMemoryMB=3992 2024-12-09T23:59:21,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T23:59:21,348 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T23:59:21,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T23:59:21,350 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T23:59:21,350 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:21,350 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 142 2024-12-09T23:59:21,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-09T23:59:21,352 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T23:59:21,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742311_1487 (size=1061) 2024-12-09T23:59:21,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-09T23:59:21,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-09T23:59:21,764 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e 2024-12-09T23:59:21,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742312_1488 (size=53) 2024-12-09T23:59:21,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-09T23:59:22,176 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:59:22,176 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f7a538750ba0f201503066ae75fe1e58, disabling compactions & flushes 2024-12-09T23:59:22,176 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:22,176 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:22,176 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. after waiting 0 ms 2024-12-09T23:59:22,176 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:22,176 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:22,176 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f7a538750ba0f201503066ae75fe1e58: Waiting for close lock at 1733788762176Disabling compacts and flushes for region at 1733788762176Disabling writes for close at 1733788762176Writing region close event to WAL at 1733788762176Closed at 1733788762176 2024-12-09T23:59:22,179 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T23:59:22,180 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733788762180"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733788762180"}]},"ts":"1733788762180"} 2024-12-09T23:59:22,183 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T23:59:22,185 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T23:59:22,185 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733788762185"}]},"ts":"1733788762185"} 2024-12-09T23:59:22,186 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T23:59:22,187 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7a538750ba0f201503066ae75fe1e58, ASSIGN}] 2024-12-09T23:59:22,188 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7a538750ba0f201503066ae75fe1e58, ASSIGN 2024-12-09T23:59:22,189 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7a538750ba0f201503066ae75fe1e58, ASSIGN; state=OFFLINE, location=2907d75fbb3e,37627,1733788599837; forceNewPlan=false, retain=false 2024-12-09T23:59:22,340 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=f7a538750ba0f201503066ae75fe1e58, regionState=OPENING, regionLocation=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:22,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7a538750ba0f201503066ae75fe1e58, ASSIGN because future has completed 2024-12-09T23:59:22,345 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837}] 2024-12-09T23:59:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-09T23:59:22,506 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:22,506 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7752): Opening region: {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} 2024-12-09T23:59:22,507 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:22,507 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:59:22,507 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7794): checking encryption for f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:22,507 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7797): checking classloading for f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:22,510 INFO [StoreOpener-f7a538750ba0f201503066ae75fe1e58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:22,513 INFO [StoreOpener-f7a538750ba0f201503066ae75fe1e58-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T23:59:22,513 INFO [StoreOpener-f7a538750ba0f201503066ae75fe1e58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7a538750ba0f201503066ae75fe1e58 columnFamilyName A 2024-12-09T23:59:22,513 DEBUG [StoreOpener-f7a538750ba0f201503066ae75fe1e58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:22,514 INFO [StoreOpener-f7a538750ba0f201503066ae75fe1e58-1 {}] regionserver.HStore(327): Store=f7a538750ba0f201503066ae75fe1e58/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:59:22,514 INFO [StoreOpener-f7a538750ba0f201503066ae75fe1e58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:22,516 INFO [StoreOpener-f7a538750ba0f201503066ae75fe1e58-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T23:59:22,516 INFO [StoreOpener-f7a538750ba0f201503066ae75fe1e58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7a538750ba0f201503066ae75fe1e58 columnFamilyName B 2024-12-09T23:59:22,516 DEBUG [StoreOpener-f7a538750ba0f201503066ae75fe1e58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:22,517 INFO [StoreOpener-f7a538750ba0f201503066ae75fe1e58-1 {}] regionserver.HStore(327): Store=f7a538750ba0f201503066ae75fe1e58/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:59:22,517 INFO [StoreOpener-f7a538750ba0f201503066ae75fe1e58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:22,518 INFO [StoreOpener-f7a538750ba0f201503066ae75fe1e58-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T23:59:22,518 INFO [StoreOpener-f7a538750ba0f201503066ae75fe1e58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7a538750ba0f201503066ae75fe1e58 columnFamilyName C 2024-12-09T23:59:22,519 DEBUG [StoreOpener-f7a538750ba0f201503066ae75fe1e58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:22,519 INFO [StoreOpener-f7a538750ba0f201503066ae75fe1e58-1 {}] regionserver.HStore(327): Store=f7a538750ba0f201503066ae75fe1e58/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:59:22,519 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:22,519 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1038): replaying wal for f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:22,520 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:22,521 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:22,522 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1048): stopping wal replay for f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:22,522 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1060): Cleaning up temporary data for f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:22,523 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T23:59:22,524 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): writing seq id for f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:22,526 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T23:59:22,527 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1114): Opened f7a538750ba0f201503066ae75fe1e58; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68740920, jitterRate=0.02431952953338623}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T23:59:22,527 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:22,528 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1006): Region open journal for f7a538750ba0f201503066ae75fe1e58: Running coprocessor pre-open hook at 1733788762507Writing region info on filesystem at 1733788762507Initializing all the Stores at 1733788762509 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788762509Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788762510 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788762510Cleaning up temporary data from old regions at 1733788762522 (+12 ms)Running coprocessor post-open hooks at 1733788762527 (+5 ms)Region opened successfully at 1733788762528 (+1 ms) 2024-12-09T23:59:22,528 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., pid=144, masterSystemTime=1733788762499 2024-12-09T23:59:22,530 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:22,530 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:22,531 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=f7a538750ba0f201503066ae75fe1e58, regionState=OPEN, openSeqNum=2, regionLocation=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:22,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 because future has completed 2024-12-09T23:59:22,537 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-12-09T23:59:22,537 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 in 190 msec 2024-12-09T23:59:22,541 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-12-09T23:59:22,541 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7a538750ba0f201503066ae75fe1e58, ASSIGN in 350 msec 2024-12-09T23:59:22,542 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T23:59:22,542 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733788762542"}]},"ts":"1733788762542"} 2024-12-09T23:59:22,544 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T23:59:22,544 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T23:59:22,546 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1960 sec 2024-12-09T23:59:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-09T23:59:23,495 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:59:23,497 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2aa61c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:59:23,497 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:59:23,497 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:59:23,500 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:59:23,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:59:23,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:59:23,501 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@438e7835, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:59:23,501 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:59:23,501 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:59:23,502 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:23,503 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37934, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:59:23,505 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@321765ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:59:23,506 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:59:23,507 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:59:23,507 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:59:23,508 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40390, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:59:23,509 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T23:59:23,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 2907d75fbb3e,45191,1733788599119 2024-12-09T23:59:23,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@54712acb 2024-12-09T23:59:23,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T23:59:23,511 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37950, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T23:59:23,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T23:59:23,512 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T23:59:23,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-09T23:59:23,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742313_1489 (size=1097) 2024-12-09T23:59:23,927 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-09T23:59:23,927 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-09T23:59:23,934 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T23:59:23,935 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7a538750ba0f201503066ae75fe1e58, REOPEN/MOVE}] 2024-12-09T23:59:23,936 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7a538750ba0f201503066ae75fe1e58, REOPEN/MOVE 2024-12-09T23:59:23,937 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=f7a538750ba0f201503066ae75fe1e58, regionState=CLOSING, regionLocation=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:23,938 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7a538750ba0f201503066ae75fe1e58, REOPEN/MOVE because future has completed 2024-12-09T23:59:23,938 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T23:59:23,938 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837}] 2024-12-09T23:59:24,091 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:24,091 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T23:59:24,092 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing f7a538750ba0f201503066ae75fe1e58, disabling compactions & flushes 2024-12-09T23:59:24,092 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:24,092 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:24,092 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. after waiting 0 ms 2024-12-09T23:59:24,092 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:24,102 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T23:59:24,103 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:24,103 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for f7a538750ba0f201503066ae75fe1e58: Waiting for close lock at 1733788764092Running coprocessor pre-close hooks at 1733788764092Disabling compacts and flushes for region at 1733788764092Disabling writes for close at 1733788764092Writing region close event to WAL at 1733788764093 (+1 ms)Running coprocessor post-close hooks at 1733788764103 (+10 ms)Closed at 1733788764103 2024-12-09T23:59:24,103 WARN [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionServer(3299): Not adding moved region record: f7a538750ba0f201503066ae75fe1e58 to self. 2024-12-09T23:59:24,106 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=f7a538750ba0f201503066ae75fe1e58, regionState=CLOSED 2024-12-09T23:59:24,106 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:24,107 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 because future has completed 2024-12-09T23:59:24,110 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-12-09T23:59:24,110 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 in 170 msec 2024-12-09T23:59:24,111 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7a538750ba0f201503066ae75fe1e58, REOPEN/MOVE; state=CLOSED, location=2907d75fbb3e,37627,1733788599837; forceNewPlan=false, retain=true 2024-12-09T23:59:24,262 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=f7a538750ba0f201503066ae75fe1e58, regionState=OPENING, regionLocation=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:24,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7a538750ba0f201503066ae75fe1e58, REOPEN/MOVE because future has completed 2024-12-09T23:59:24,267 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837}] 2024-12-09T23:59:24,428 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:24,428 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7752): Opening region: {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} 2024-12-09T23:59:24,428 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:24,429 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:59:24,429 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7794): checking encryption for f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:24,429 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7797): checking classloading for f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:24,432 INFO [StoreOpener-f7a538750ba0f201503066ae75fe1e58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:24,433 INFO [StoreOpener-f7a538750ba0f201503066ae75fe1e58-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T23:59:24,433 INFO [StoreOpener-f7a538750ba0f201503066ae75fe1e58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7a538750ba0f201503066ae75fe1e58 columnFamilyName A 2024-12-09T23:59:24,436 DEBUG [StoreOpener-f7a538750ba0f201503066ae75fe1e58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:24,436 INFO [StoreOpener-f7a538750ba0f201503066ae75fe1e58-1 {}] regionserver.HStore(327): Store=f7a538750ba0f201503066ae75fe1e58/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:59:24,437 INFO [StoreOpener-f7a538750ba0f201503066ae75fe1e58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:24,438 INFO [StoreOpener-f7a538750ba0f201503066ae75fe1e58-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T23:59:24,438 INFO [StoreOpener-f7a538750ba0f201503066ae75fe1e58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7a538750ba0f201503066ae75fe1e58 columnFamilyName B 2024-12-09T23:59:24,438 DEBUG [StoreOpener-f7a538750ba0f201503066ae75fe1e58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:24,439 INFO [StoreOpener-f7a538750ba0f201503066ae75fe1e58-1 {}] regionserver.HStore(327): Store=f7a538750ba0f201503066ae75fe1e58/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:59:24,439 INFO [StoreOpener-f7a538750ba0f201503066ae75fe1e58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:24,440 INFO [StoreOpener-f7a538750ba0f201503066ae75fe1e58-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T23:59:24,440 INFO [StoreOpener-f7a538750ba0f201503066ae75fe1e58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7a538750ba0f201503066ae75fe1e58 columnFamilyName C 2024-12-09T23:59:24,440 DEBUG [StoreOpener-f7a538750ba0f201503066ae75fe1e58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:24,441 INFO [StoreOpener-f7a538750ba0f201503066ae75fe1e58-1 {}] regionserver.HStore(327): Store=f7a538750ba0f201503066ae75fe1e58/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:59:24,441 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:24,441 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1038): replaying wal for f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:24,442 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:24,444 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:24,446 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1048): stopping wal replay for f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:24,447 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1060): Cleaning up temporary data for f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:24,447 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T23:59:24,450 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1093): writing seq id for f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:24,451 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1114): Opened f7a538750ba0f201503066ae75fe1e58; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68672872, jitterRate=0.023305535316467285}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T23:59:24,452 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:24,453 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1006): Region open journal for f7a538750ba0f201503066ae75fe1e58: Running coprocessor pre-open hook at 1733788764429Writing region info on filesystem at 1733788764429Initializing all the Stores at 1733788764430 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788764431 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788764431Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733788764431Cleaning up temporary data from old regions at 1733788764447 (+16 ms)Running coprocessor post-open hooks at 1733788764452 (+5 ms)Region opened successfully at 1733788764453 (+1 ms) 2024-12-09T23:59:24,454 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., pid=149, masterSystemTime=1733788764423 2024-12-09T23:59:24,457 DEBUG [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:24,457 INFO [RS_OPEN_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:24,457 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=f7a538750ba0f201503066ae75fe1e58, regionState=OPEN, openSeqNum=5, regionLocation=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:24,459 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 because future has completed 2024-12-09T23:59:24,462 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=147 2024-12-09T23:59:24,462 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 in 193 msec 2024-12-09T23:59:24,463 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-12-09T23:59:24,463 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7a538750ba0f201503066ae75fe1e58, REOPEN/MOVE in 527 msec 2024-12-09T23:59:24,465 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-12-09T23:59:24,465 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 530 msec 2024-12-09T23:59:24,467 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 953 msec 2024-12-09T23:59:24,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-09T23:59:24,468 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:59:24,468 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fc022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:59:24,468 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:59:24,468 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:59:24,469 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:59:24,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:59:24,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:59:24,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eb7797, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:59:24,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:59:24,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:59:24,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:24,470 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37976, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:59:24,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cd0a2ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:59:24,471 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77590f25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:59:24,471 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:59:24,471 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:59:24,472 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:59:24,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:59:24,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:59:24,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@953cba8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:59:24,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:59:24,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:59:24,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:24,473 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37984, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:59:24,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e44d9c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:59:24,474 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99b73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:59:24,474 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:59:24,474 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:59:24,475 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:59:24,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:59:24,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:59:24,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@581a3208, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:59:24,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:59:24,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:59:24,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:24,476 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38008, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:59:24,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d55b9c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:59:24,477 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af69a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:59:24,477 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:59:24,477 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:59:24,478 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:59:24,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:59:24,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:59:24,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d65c626, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:59:24,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:59:24,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:59:24,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:24,479 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38028, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:59:24,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36ab6131, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:59:24,480 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd90259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:59:24,480 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:59:24,480 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:59:24,481 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:59:24,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:59:24,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:59:24,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62d56670, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:59:24,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:59:24,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:59:24,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:24,483 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38058, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:59:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6822ab36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:59:24,485 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ae597b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:59:24,485 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:59:24,485 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:59:24,485 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:59:24,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:59:24,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:59:24,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@470b4df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:59:24,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:59:24,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:59:24,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:24,487 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38082, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:59:24,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55d697e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:59:24,488 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39174ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:59:24,488 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:59:24,488 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:59:24,489 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:59:24,489 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:59:24,489 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:59:24,489 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2738f5cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:59:24,489 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:59:24,489 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:59:24,490 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:24,490 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38096, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:59:24,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dfa087f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:59:24,492 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8fc71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:59:24,492 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:59:24,492 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:59:24,492 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:59:24,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:59:24,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:59:24,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f98db7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:59:24,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:59:24,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:59:24,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:24,494 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38120, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:59:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fb9ff6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:59:24,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dc96918, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:59:24,495 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:59:24,495 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:59:24,496 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:59:24,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:59:24,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:59:24,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@190347ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:59:24,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:59:24,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:59:24,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:24,497 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38138, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:59:24,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@407d1ac9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:59:24,498 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70d22de7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:59:24,499 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2907d75fbb3e,45191,-1 for getting cluster id 2024-12-09T23:59:24,499 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:59:24,499 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbe338b0-2c12-4f81-bb85-08f4438f065e' 2024-12-09T23:59:24,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:59:24,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbe338b0-2c12-4f81-bb85-08f4438f065e" 2024-12-09T23:59:24,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45036fde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:59:24,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2907d75fbb3e,45191,-1] 2024-12-09T23:59:24,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:59:24,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:24,500 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38156, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:59:24,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dbcbe93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:59:24,501 DEBUG [Thread-2207 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:59:24,501 DEBUG [Thread-2209 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:59:24,501 DEBUG [Thread-2226 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:59:24,501 DEBUG [Thread-2215 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:59:24,501 DEBUG [Thread-2224 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:59:24,501 DEBUG [Thread-2220 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:59:24,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:59:24,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:59:24,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:59:24,502 DEBUG [Thread-2213 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:59:24,502 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:59:24,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:59:24,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:59:24,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:59:24,502 DEBUG [Thread-2222 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:59:24,502 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:59:24,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:59:24,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:59:24,502 DEBUG [Thread-2218 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:59:24,503 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:59:24,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:59:24,503 DEBUG [Thread-2211 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:59:24,503 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40398, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:59:24,503 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:59:24,503 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40410, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:59:24,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:59:24,503 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40434, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:59:24,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:59:24,503 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40446, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:59:24,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:59:24,504 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40400, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:59:24,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:59:24,504 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40424, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:59:24,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:59:24,504 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40458, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:59:24,504 DEBUG [Thread-2218 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:59:24,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-12-09T23:59:24,505 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40474, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:59:24,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5] 2024-12-09T23:59:24,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5] 2024-12-09T23:59:24,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-09T23:59:24,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2907d75fbb3e,37627,1733788599837, seqNum=-1] 2024-12-09T23:59:24,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:59:24,506 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:59:24,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5] 2024-12-09T23:59:24,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5] 2024-12-09T23:59:24,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5] 2024-12-09T23:59:24,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5] 2024-12-09T23:59:24,507 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:59:24,507 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40488, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:59:24,507 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:59:24,508 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5] 2024-12-09T23:59:24,508 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40504, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:59:24,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5] 2024-12-09T23:59:24,508 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5] 2024-12-09T23:59:24,510 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5] 2024-12-09T23:59:24,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:24,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7a538750ba0f201503066ae75fe1e58 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T23:59:24,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=A 2024-12-09T23:59:24,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:24,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=B 2024-12-09T23:59:24,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:24,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=C 2024-12-09T23:59:24,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:24,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c60f4e7fa81f4c75857d11e7c695a6cc_f7a538750ba0f201503066ae75fe1e58 is 50, key is test_row_0/A:col10/1733788764512/Put/seqid=0 2024-12-09T23:59:24,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742314_1490 (size=12154) 2024-12-09T23:59:24,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:24,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788824547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788824548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:24,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40458 deadline: 1733788824548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:24,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:24,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:24,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:24,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:24,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:24,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:24,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788824551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:24,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:24,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788824552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:24,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:24,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:24,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:24,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:24,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:24,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:24,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:24,555 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:24,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-09T23:59:24,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T23:59:24,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:24,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:24,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:24,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:24,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:24,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:24,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788824663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:24,664 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:24,664 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:24,664 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:24,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:24,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:24,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788824666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:24,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:24,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788824667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:24,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:24,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:24,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:24,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:24,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:24,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:24,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:24,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40458 deadline: 1733788824667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:24,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:24,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788824667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:24,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:24,668 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:24,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:24,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:24,668 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:24,668 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:24,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T23:59:24,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:24,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:24,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:24,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:24,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:24,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-09T23:59:24,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788824872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:24,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:24,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:24,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:24,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788824872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:24,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:24,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:24,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:24,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788824873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:24,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788824873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:24,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40458 deadline: 1733788824873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:24,934 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:24,937 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c60f4e7fa81f4c75857d11e7c695a6cc_f7a538750ba0f201503066ae75fe1e58 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c60f4e7fa81f4c75857d11e7c695a6cc_f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:24,938 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/cdb04b7b249347668bdb3199534c0728, store: [table=TestAcidGuarantees family=A region=f7a538750ba0f201503066ae75fe1e58] 2024-12-09T23:59:24,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/cdb04b7b249347668bdb3199534c0728 is 175, key is test_row_0/A:col10/1733788764512/Put/seqid=0 2024-12-09T23:59:24,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742315_1491 (size=30955) 2024-12-09T23:59:24,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T23:59:24,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:24,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:24,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:24,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:24,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:24,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:25,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T23:59:25,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:25,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:25,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:25,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:25,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:25,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:25,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-09T23:59:25,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:25,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788825182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:25,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:25,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788825182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:25,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:25,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788825182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:25,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:25,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:25,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788825183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:25,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40458 deadline: 1733788825183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:25,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T23:59:25,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:25,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:25,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:25,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:25,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:25,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:25,342 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/cdb04b7b249347668bdb3199534c0728 2024-12-09T23:59:25,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/7e364e022c9b4ff2a23d3a153bc76833 is 50, key is test_row_0/B:col10/1733788764512/Put/seqid=0 2024-12-09T23:59:25,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742316_1492 (size=12001) 2024-12-09T23:59:25,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T23:59:25,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:25,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:25,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:25,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:25,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:25,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:25,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T23:59:25,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:25,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:25,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:25,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:25,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:25,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:25,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-09T23:59:25,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788825692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:25,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:25,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788825692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:25,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:25,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:25,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:25,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:25,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:25,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788825692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:25,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:25,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:25,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:25,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:25,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788825693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:25,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:25,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:25,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:25,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:25,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40458 deadline: 1733788825693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:25,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:25,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:25,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:25,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T23:59:25,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:25,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:25,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:25,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:25,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:25,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:25,762 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/7e364e022c9b4ff2a23d3a153bc76833 2024-12-09T23:59:25,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/8db7d472b0ae429dbfa4145d10d7c5b3 is 50, key is test_row_0/C:col10/1733788764512/Put/seqid=0 2024-12-09T23:59:25,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742317_1493 (size=12001) 2024-12-09T23:59:25,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T23:59:25,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:25,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:25,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:25,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:25,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:25,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:26,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T23:59:26,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:26,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:26,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:26,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:26,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:26,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:26,050 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T23:59:26,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/8db7d472b0ae429dbfa4145d10d7c5b3 2024-12-09T23:59:26,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T23:59:26,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:26,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:26,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:26,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:26,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:26,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/cdb04b7b249347668bdb3199534c0728 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/cdb04b7b249347668bdb3199534c0728 2024-12-09T23:59:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:26,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/cdb04b7b249347668bdb3199534c0728, entries=150, sequenceid=16, filesize=30.2 K 2024-12-09T23:59:26,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/7e364e022c9b4ff2a23d3a153bc76833 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/7e364e022c9b4ff2a23d3a153bc76833 2024-12-09T23:59:26,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/7e364e022c9b4ff2a23d3a153bc76833, entries=150, sequenceid=16, filesize=11.7 K 2024-12-09T23:59:26,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/8db7d472b0ae429dbfa4145d10d7c5b3 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/8db7d472b0ae429dbfa4145d10d7c5b3 2024-12-09T23:59:26,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/8db7d472b0ae429dbfa4145d10d7c5b3, entries=150, sequenceid=16, filesize=11.7 K 2024-12-09T23:59:26,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f7a538750ba0f201503066ae75fe1e58 in 1692ms, sequenceid=16, compaction requested=false 2024-12-09T23:59:26,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7a538750ba0f201503066ae75fe1e58: 2024-12-09T23:59:26,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T23:59:26,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:26,344 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing f7a538750ba0f201503066ae75fe1e58 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T23:59:26,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=A 2024-12-09T23:59:26,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:26,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=B 2024-12-09T23:59:26,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:26,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=C 2024-12-09T23:59:26,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:26,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209adce10c870ce48a98d1dc85198ae90f8_f7a538750ba0f201503066ae75fe1e58 is 50, key is test_row_0/A:col10/1733788764550/Put/seqid=0 2024-12-09T23:59:26,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742318_1494 (size=12154) 2024-12-09T23:59:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-09T23:59:26,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:26,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:26,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:26,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788826708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:26,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:26,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:26,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:26,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:26,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788826710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:26,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:26,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:26,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:26,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:26,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788826710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:26,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:26,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:26,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:26,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:26,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40458 deadline: 1733788826711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:26,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:26,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:26,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:26,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:26,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788826711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:26,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:26,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:26,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:26,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:26,755 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209adce10c870ce48a98d1dc85198ae90f8_f7a538750ba0f201503066ae75fe1e58 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209adce10c870ce48a98d1dc85198ae90f8_f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:26,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/b1348fbd47ae4af8a42c5e451985d121, store: [table=TestAcidGuarantees family=A region=f7a538750ba0f201503066ae75fe1e58] 2024-12-09T23:59:26,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/b1348fbd47ae4af8a42c5e451985d121 is 175, key is test_row_0/A:col10/1733788764550/Put/seqid=0 2024-12-09T23:59:26,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742319_1495 (size=30955) 2024-12-09T23:59:26,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:26,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788826822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:26,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:26,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:26,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:26,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:26,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:26,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788826822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:26,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788826822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:26,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:26,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:26,824 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:26,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788826823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:26,824 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:26,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40458 deadline: 1733788826823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:26,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:26,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:26,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:26,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:26,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:26,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:26,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:26,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:26,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:26,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:27,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788827032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:27,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:27,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:27,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:27,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788827032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:27,034 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:27,034 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:27,034 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:27,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788827033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:27,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:27,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:27,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:27,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:27,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40458 deadline: 1733788827033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788827033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:27,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:27,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:27,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:27,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:27,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:27,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:27,160 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/b1348fbd47ae4af8a42c5e451985d121 2024-12-09T23:59:27,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/9a5ce0fc8d4a4abba199e8fcb9ecb685 is 50, key is test_row_0/B:col10/1733788764550/Put/seqid=0 2024-12-09T23:59:27,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742320_1496 (size=12001) 2024-12-09T23:59:27,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:27,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788827343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:27,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:27,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:27,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788827343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:27,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:27,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:27,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:27,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788827343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:27,346 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:27,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:27,346 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:27,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:27,346 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:27,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:27,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40458 deadline: 1733788827343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:27,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:27,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:27,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788827343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:27,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:27,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:27,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:27,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:27,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:27,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:27,578 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/9a5ce0fc8d4a4abba199e8fcb9ecb685 2024-12-09T23:59:27,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/b6074e701b4c45e4ab177e43b66dddf1 is 50, key is test_row_0/C:col10/1733788764550/Put/seqid=0 2024-12-09T23:59:27,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742321_1497 (size=12001) 2024-12-09T23:59:27,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788827853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:27,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788827853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:27,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:27,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788827853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:27,855 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:27,855 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:27,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788827854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:27,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:27,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:27,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:27,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:27,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:27,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:27,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:27,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:27,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:27,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40458 deadline: 1733788827859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:27,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:27,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:27,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:27,986 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/b6074e701b4c45e4ab177e43b66dddf1 2024-12-09T23:59:27,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/b1348fbd47ae4af8a42c5e451985d121 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/b1348fbd47ae4af8a42c5e451985d121 2024-12-09T23:59:27,994 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/b1348fbd47ae4af8a42c5e451985d121, entries=150, sequenceid=36, filesize=30.2 K 2024-12-09T23:59:27,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/9a5ce0fc8d4a4abba199e8fcb9ecb685 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/9a5ce0fc8d4a4abba199e8fcb9ecb685 2024-12-09T23:59:27,998 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/9a5ce0fc8d4a4abba199e8fcb9ecb685, entries=150, sequenceid=36, filesize=11.7 K 2024-12-09T23:59:27,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/b6074e701b4c45e4ab177e43b66dddf1 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/b6074e701b4c45e4ab177e43b66dddf1 2024-12-09T23:59:28,002 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/b6074e701b4c45e4ab177e43b66dddf1, entries=150, sequenceid=36, filesize=11.7 K 2024-12-09T23:59:28,003 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for f7a538750ba0f201503066ae75fe1e58 in 1659ms, sequenceid=36, compaction requested=false 2024-12-09T23:59:28,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for f7a538750ba0f201503066ae75fe1e58: 2024-12-09T23:59:28,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:28,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-12-09T23:59:28,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-12-09T23:59:28,007 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-12-09T23:59:28,007 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4980 sec 2024-12-09T23:59:28,008 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 3.5040 sec 2024-12-09T23:59:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-09T23:59:28,663 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:59:28,663 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:59:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-09T23:59:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T23:59:28,664 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:59:28,665 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:59:28,665 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:59:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T23:59:28,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T23:59:28,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:28,817 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing f7a538750ba0f201503066ae75fe1e58 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T23:59:28,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=A 2024-12-09T23:59:28,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:28,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=B 2024-12-09T23:59:28,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:28,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=C 2024-12-09T23:59:28,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:28,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412091b2e5008e1ca45e5b7ffddbdd14f506c_f7a538750ba0f201503066ae75fe1e58 is 50, key is test_row_0/A:col10/1733788766709/Put/seqid=0 2024-12-09T23:59:28,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742322_1498 (size=12154) 2024-12-09T23:59:28,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:28,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:28,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:28,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788828878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:28,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:28,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:28,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:28,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:28,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788828880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:28,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:28,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:28,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788828880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:28,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:28,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:28,883 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:28,883 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:28,883 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:28,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:28,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788828882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:28,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:28,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:28,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40458 deadline: 1733788828882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:28,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:28,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:28,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:28,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:28,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:28,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T23:59:28,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:28,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788828993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:28,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:28,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788828993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:28,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:28,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:28,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:28,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:28,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788828993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:28,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:28,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:28,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788828993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:28,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:28,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:28,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:28,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:28,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:28,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:28,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40458 deadline: 1733788828993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:28,995 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:28,995 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:28,995 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:29,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788829203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:29,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:29,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:29,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:29,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788829203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:29,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:29,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:29,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:29,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788829203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:29,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:29,206 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788829203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:29,206 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:29,206 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:29,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:29,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:29,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:29,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40458 deadline: 1733788829203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:29,207 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:29,207 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:29,207 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:29,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:29,230 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412091b2e5008e1ca45e5b7ffddbdd14f506c_f7a538750ba0f201503066ae75fe1e58 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091b2e5008e1ca45e5b7ffddbdd14f506c_f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:29,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/8d994d3917f941f28358d4e495c9da79, store: [table=TestAcidGuarantees family=A region=f7a538750ba0f201503066ae75fe1e58] 2024-12-09T23:59:29,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/8d994d3917f941f28358d4e495c9da79 is 175, key is test_row_0/A:col10/1733788766709/Put/seqid=0 2024-12-09T23:59:29,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742323_1499 (size=30955) 2024-12-09T23:59:29,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T23:59:29,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788829513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:29,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:29,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:29,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788829513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:29,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788829513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:29,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:29,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:29,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:29,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:29,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:29,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:29,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:29,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40458 deadline: 1733788829513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:29,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788829513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:29,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:29,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:29,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:29,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:29,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:29,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:29,636 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/8d994d3917f941f28358d4e495c9da79 2024-12-09T23:59:29,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/db3e6e74422b485fbd370937be3585dd is 50, key is test_row_0/B:col10/1733788766709/Put/seqid=0 2024-12-09T23:59:29,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742324_1500 (size=12001) 2024-12-09T23:59:29,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T23:59:30,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:30,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788830023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:30,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:30,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:30,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:30,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:30,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788830023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:30,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:30,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788830023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:30,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:30,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788830023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:30,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:30,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40458 deadline: 1733788830023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:30,046 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/db3e6e74422b485fbd370937be3585dd 2024-12-09T23:59:30,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/b5c386ca3be94452ab94548b45eb9f4f is 50, key is test_row_0/C:col10/1733788766709/Put/seqid=0 2024-12-09T23:59:30,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742325_1501 (size=12001) 2024-12-09T23:59:30,455 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/b5c386ca3be94452ab94548b45eb9f4f 2024-12-09T23:59:30,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/8d994d3917f941f28358d4e495c9da79 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/8d994d3917f941f28358d4e495c9da79 2024-12-09T23:59:30,462 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/8d994d3917f941f28358d4e495c9da79, entries=150, sequenceid=49, filesize=30.2 K 2024-12-09T23:59:30,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/db3e6e74422b485fbd370937be3585dd as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/db3e6e74422b485fbd370937be3585dd 2024-12-09T23:59:30,466 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/db3e6e74422b485fbd370937be3585dd, entries=150, sequenceid=49, filesize=11.7 K 2024-12-09T23:59:30,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/b5c386ca3be94452ab94548b45eb9f4f as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/b5c386ca3be94452ab94548b45eb9f4f 2024-12-09T23:59:30,470 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/b5c386ca3be94452ab94548b45eb9f4f, entries=150, sequenceid=49, filesize=11.7 K 2024-12-09T23:59:30,470 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f7a538750ba0f201503066ae75fe1e58 in 1653ms, sequenceid=49, compaction requested=true 2024-12-09T23:59:30,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for f7a538750ba0f201503066ae75fe1e58: 2024-12-09T23:59:30,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:30,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-09T23:59:30,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-09T23:59:30,473 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-09T23:59:30,473 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8070 sec 2024-12-09T23:59:30,474 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.8100 sec 2024-12-09T23:59:30,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T23:59:30,813 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:59:30,813 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:59:30,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-09T23:59:30,815 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:59:30,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T23:59:30,815 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:59:30,815 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:59:30,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T23:59:30,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-09T23:59:30,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:30,973 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing f7a538750ba0f201503066ae75fe1e58 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T23:59:30,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=A 2024-12-09T23:59:30,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:30,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=B 2024-12-09T23:59:30,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:30,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=C 2024-12-09T23:59:30,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:30,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099945818e2a45425d84541785f4348dee_f7a538750ba0f201503066ae75fe1e58 is 50, key is test_row_0/A:col10/1733788768881/Put/seqid=0 2024-12-09T23:59:30,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742326_1502 (size=12154) 2024-12-09T23:59:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:31,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:31,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788831040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:31,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788831040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:31,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788831041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:31,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:31,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788831042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:31,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:31,046 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:31,046 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,046 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:31,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40458 deadline: 1733788831042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:31,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T23:59:31,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788831153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:31,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788831153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:31,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:31,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:31,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788831153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:31,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:31,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:31,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788831153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:31,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:31,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40458 deadline: 1733788831153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:31,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:31,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:31,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788831363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:31,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:31,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788831363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:31,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:31,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:31,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788831363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:31,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:31,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:31,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:31,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40458 deadline: 1733788831364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:31,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:31,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:31,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788831364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:31,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:31,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:31,385 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099945818e2a45425d84541785f4348dee_f7a538750ba0f201503066ae75fe1e58 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099945818e2a45425d84541785f4348dee_f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:31,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/77ff084e8b274127aba6844500da0c52, store: [table=TestAcidGuarantees family=A region=f7a538750ba0f201503066ae75fe1e58] 2024-12-09T23:59:31,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/77ff084e8b274127aba6844500da0c52 is 175, key is test_row_0/A:col10/1733788768881/Put/seqid=0 2024-12-09T23:59:31,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742327_1503 (size=30955) 2024-12-09T23:59:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T23:59:31,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:31,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788831673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:31,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:31,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788831673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:31,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:31,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:31,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:31,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40458 deadline: 1733788831673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:31,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:31,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:31,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788831673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:31,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:31,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788831673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:31,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:31,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:31,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:31,790 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=68, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/77ff084e8b274127aba6844500da0c52 2024-12-09T23:59:31,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/abb6b565b4f444c4b1bd65b299ec7ab5 is 50, key is test_row_0/B:col10/1733788768881/Put/seqid=0 2024-12-09T23:59:31,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742328_1504 (size=12001) 2024-12-09T23:59:31,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T23:59:32,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:32,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788832183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:32,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:32,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:32,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:32,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788832183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:32,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:32,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:32,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40458 deadline: 1733788832183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:32,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:32,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:32,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:32,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:32,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788832183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:32,186 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:32,186 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:32,186 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:32,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:32,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:32,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788832183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:32,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:32,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:32,186 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:32,186 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:32,186 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:32,200 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/abb6b565b4f444c4b1bd65b299ec7ab5 2024-12-09T23:59:32,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/ab6a8e90cb2d4247ad32fbe488d632f4 is 50, key is test_row_0/C:col10/1733788768881/Put/seqid=0 2024-12-09T23:59:32,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742329_1505 (size=12001) 2024-12-09T23:59:32,609 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/ab6a8e90cb2d4247ad32fbe488d632f4 2024-12-09T23:59:32,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/77ff084e8b274127aba6844500da0c52 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/77ff084e8b274127aba6844500da0c52 2024-12-09T23:59:32,616 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/77ff084e8b274127aba6844500da0c52, entries=150, sequenceid=68, filesize=30.2 K 2024-12-09T23:59:32,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/abb6b565b4f444c4b1bd65b299ec7ab5 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/abb6b565b4f444c4b1bd65b299ec7ab5 2024-12-09T23:59:32,619 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/abb6b565b4f444c4b1bd65b299ec7ab5, entries=150, sequenceid=68, filesize=11.7 K 2024-12-09T23:59:32,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/ab6a8e90cb2d4247ad32fbe488d632f4 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/ab6a8e90cb2d4247ad32fbe488d632f4 2024-12-09T23:59:32,623 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/ab6a8e90cb2d4247ad32fbe488d632f4, entries=150, sequenceid=68, filesize=11.7 K 2024-12-09T23:59:32,624 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for f7a538750ba0f201503066ae75fe1e58 in 1651ms, sequenceid=68, compaction requested=true 2024-12-09T23:59:32,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for f7a538750ba0f201503066ae75fe1e58: 2024-12-09T23:59:32,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:32,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-09T23:59:32,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-09T23:59:32,626 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-09T23:59:32,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8100 sec 2024-12-09T23:59:32,628 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.8130 sec 2024-12-09T23:59:32,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T23:59:32,963 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:59:32,963 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:59:32,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-09T23:59:32,965 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:59:32,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-09T23:59:32,965 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:59:32,965 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:59:33,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-09T23:59:33,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-09T23:59:33,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:33,117 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing f7a538750ba0f201503066ae75fe1e58 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T23:59:33,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=A 2024-12-09T23:59:33,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:33,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=B 2024-12-09T23:59:33,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:33,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=C 2024-12-09T23:59:33,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:33,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209db304e03ce8540d2a0f090daa3d9135d_f7a538750ba0f201503066ae75fe1e58 is 50, key is test_row_0/A:col10/1733788771039/Put/seqid=0 2024-12-09T23:59:33,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742330_1506 (size=12154) 2024-12-09T23:59:33,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:33,127 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209db304e03ce8540d2a0f090daa3d9135d_f7a538750ba0f201503066ae75fe1e58 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209db304e03ce8540d2a0f090daa3d9135d_f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:33,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/e0aaafe4fc1a4cfdb9ff2c6b9acb4ffc, store: [table=TestAcidGuarantees family=A region=f7a538750ba0f201503066ae75fe1e58] 2024-12-09T23:59:33,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/e0aaafe4fc1a4cfdb9ff2c6b9acb4ffc is 175, key is test_row_0/A:col10/1733788771039/Put/seqid=0 2024-12-09T23:59:33,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742331_1507 (size=30955) 2024-12-09T23:59:33,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:33,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:33,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:33,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788833212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:33,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788833212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:33,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788833213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:33,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788833214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:33,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:33,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:33,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40458 deadline: 1733788833216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:33,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,220 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:33,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-09T23:59:33,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:33,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788833322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:33,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:33,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:33,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788833322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:33,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:33,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:33,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788833323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:33,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:33,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:33,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788833323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:33,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:33,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:33,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40458 deadline: 1733788833323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:33,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:33,531 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/e0aaafe4fc1a4cfdb9ff2c6b9acb4ffc 2024-12-09T23:59:33,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:33,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788833532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:33,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:33,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788833532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:33,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:33,535 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:33,535 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,536 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:33,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788833533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:33,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:33,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:33,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788833533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:33,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:33,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:33,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40458 deadline: 1733788833533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:33,536 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,536 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,536 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:33,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/7a3667c0c3224003bef6949a53c4ad19 is 50, key is test_row_0/B:col10/1733788771039/Put/seqid=0 2024-12-09T23:59:33,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742332_1508 (size=12001) 2024-12-09T23:59:33,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-09T23:59:33,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:33,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788833842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:33,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:33,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:33,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788833842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:33,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:33,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:33,845 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788833842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:33,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788833842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:33,845 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,845 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:33,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:33,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:33,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40458 deadline: 1733788833842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:33,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:33,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:33,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:33,940 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/7a3667c0c3224003bef6949a53c4ad19 2024-12-09T23:59:33,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/c2b03ca56f6b4eebaa27320fef402aa4 is 50, key is test_row_0/C:col10/1733788771039/Put/seqid=0 2024-12-09T23:59:33,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742333_1509 (size=12001) 2024-12-09T23:59:34,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-09T23:59:34,349 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/c2b03ca56f6b4eebaa27320fef402aa4 2024-12-09T23:59:34,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:34,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/e0aaafe4fc1a4cfdb9ff2c6b9acb4ffc as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/e0aaafe4fc1a4cfdb9ff2c6b9acb4ffc 2024-12-09T23:59:34,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788834353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:34,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:34,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:34,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788834353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:34,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:34,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:34,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:34,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:34,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:34,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:34,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40458 deadline: 1733788834353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:34,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:34,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:34,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:34,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:34,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788834353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:34,355 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:34,355 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:34,355 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:34,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:34,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788834353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:34,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:34,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:34,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:34,358 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/e0aaafe4fc1a4cfdb9ff2c6b9acb4ffc, entries=150, sequenceid=81, filesize=30.2 K 2024-12-09T23:59:34,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/7a3667c0c3224003bef6949a53c4ad19 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/7a3667c0c3224003bef6949a53c4ad19 2024-12-09T23:59:34,361 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/7a3667c0c3224003bef6949a53c4ad19, entries=150, sequenceid=81, filesize=11.7 K 2024-12-09T23:59:34,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/c2b03ca56f6b4eebaa27320fef402aa4 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/c2b03ca56f6b4eebaa27320fef402aa4 2024-12-09T23:59:34,364 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/c2b03ca56f6b4eebaa27320fef402aa4, entries=150, sequenceid=81, filesize=11.7 K 2024-12-09T23:59:34,365 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f7a538750ba0f201503066ae75fe1e58 in 1248ms, sequenceid=81, compaction requested=true 2024-12-09T23:59:34,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for f7a538750ba0f201503066ae75fe1e58: 2024-12-09T23:59:34,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:34,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-09T23:59:34,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-09T23:59:34,367 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-09T23:59:34,367 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4010 sec 2024-12-09T23:59:34,368 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.4040 sec 2024-12-09T23:59:35,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-09T23:59:35,113 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:59:35,114 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:59:35,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-09T23:59:35,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-09T23:59:35,115 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:59:35,116 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:59:35,116 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:59:35,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-09T23:59:35,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-09T23:59:35,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:35,268 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing f7a538750ba0f201503066ae75fe1e58 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T23:59:35,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=A 2024-12-09T23:59:35,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:35,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=B 2024-12-09T23:59:35,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:35,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=C 2024-12-09T23:59:35,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:35,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099ba197c7d2164053909619d57a6ea6ff_f7a538750ba0f201503066ae75fe1e58 is 50, key is test_row_0/A:col10/1733788773212/Put/seqid=0 2024-12-09T23:59:35,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742334_1510 (size=12154) 2024-12-09T23:59:35,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:35,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:35,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:35,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788835371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:35,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:35,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788835371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:35,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:35,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:35,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:35,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:35,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:35,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:35,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:35,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40458 deadline: 1733788835372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:35,376 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_0' in f7a538750ba0f201503066ae75fe1e58 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:59:35,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:35,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:35,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:35,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:35,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788835375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:35,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:35,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:35,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:35,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:35,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788835375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:35,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:35,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:35,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:35,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-09T23:59:35,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:35,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788835482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:35,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:35,485 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:35,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788835483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:35,485 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:35,485 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:35,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:35,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:35,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:35,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:35,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788835483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:35,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:35,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:35,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:35,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:35,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788835483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:35,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:35,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:35,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:35,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:35,680 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099ba197c7d2164053909619d57a6ea6ff_f7a538750ba0f201503066ae75fe1e58 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099ba197c7d2164053909619d57a6ea6ff_f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:35,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/9168cd5a74294c24b56416abce10a5d3, store: [table=TestAcidGuarantees family=A region=f7a538750ba0f201503066ae75fe1e58] 2024-12-09T23:59:35,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/9168cd5a74294c24b56416abce10a5d3 is 175, key is test_row_0/A:col10/1733788773212/Put/seqid=0 2024-12-09T23:59:35,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742335_1511 (size=30955) 2024-12-09T23:59:35,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:35,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788835692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:35,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:35,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:35,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:35,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788835693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:35,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788835693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:35,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:35,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:35,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:35,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788835693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:35,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:35,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:35,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:35,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:35,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:35,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:35,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-09T23:59:36,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:36,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788836003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:36,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:36,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:36,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788836003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:36,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:36,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:36,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:36,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:36,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:36,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:36,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788836003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:36,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:36,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:36,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:36,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:36,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788836003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:36,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:36,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:36,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:36,086 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=100, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/9168cd5a74294c24b56416abce10a5d3 2024-12-09T23:59:36,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/9f4e399827d64e79b1f697d3f310e0b4 is 50, key is test_row_0/B:col10/1733788773212/Put/seqid=0 2024-12-09T23:59:36,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742336_1512 (size=12001) 2024-12-09T23:59:36,115 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/9f4e399827d64e79b1f697d3f310e0b4 2024-12-09T23:59:36,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/e046d140922c40a9a20255cccd2cac30 is 50, key is test_row_0/C:col10/1733788773212/Put/seqid=0 2024-12-09T23:59:36,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742337_1513 (size=12001) 2024-12-09T23:59:36,124 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/e046d140922c40a9a20255cccd2cac30 2024-12-09T23:59:36,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/9168cd5a74294c24b56416abce10a5d3 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/9168cd5a74294c24b56416abce10a5d3 2024-12-09T23:59:36,130 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/9168cd5a74294c24b56416abce10a5d3, entries=150, sequenceid=100, filesize=30.2 K 2024-12-09T23:59:36,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/9f4e399827d64e79b1f697d3f310e0b4 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/9f4e399827d64e79b1f697d3f310e0b4 2024-12-09T23:59:36,134 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/9f4e399827d64e79b1f697d3f310e0b4, entries=150, sequenceid=100, filesize=11.7 K 2024-12-09T23:59:36,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/e046d140922c40a9a20255cccd2cac30 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/e046d140922c40a9a20255cccd2cac30 2024-12-09T23:59:36,137 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/e046d140922c40a9a20255cccd2cac30, entries=150, sequenceid=100, filesize=11.7 K 2024-12-09T23:59:36,138 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for f7a538750ba0f201503066ae75fe1e58 in 870ms, sequenceid=100, compaction requested=true 2024-12-09T23:59:36,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for f7a538750ba0f201503066ae75fe1e58: 2024-12-09T23:59:36,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:36,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-09T23:59:36,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-09T23:59:36,140 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-09T23:59:36,140 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0230 sec 2024-12-09T23:59:36,141 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.0270 sec 2024-12-09T23:59:36,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-09T23:59:36,253 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:59:36,254 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:59:36,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-12-09T23:59:36,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-09T23:59:36,255 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:59:36,255 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:59:36,256 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:59:36,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-09T23:59:36,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-09T23:59:36,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:36,407 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing f7a538750ba0f201503066ae75fe1e58 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T23:59:36,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=A 2024-12-09T23:59:36,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:36,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=B 2024-12-09T23:59:36,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:36,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=C 2024-12-09T23:59:36,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:36,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120924d87fa53b924c1d80d174b7eb801eac_f7a538750ba0f201503066ae75fe1e58 is 50, key is test_row_0/A:col10/1733788775374/Put/seqid=0 2024-12-09T23:59:36,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742338_1514 (size=12154) 2024-12-09T23:59:36,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:36,420 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120924d87fa53b924c1d80d174b7eb801eac_f7a538750ba0f201503066ae75fe1e58 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120924d87fa53b924c1d80d174b7eb801eac_f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:36,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/101b6a16ccd94c279d658f083a6e7be4, store: [table=TestAcidGuarantees family=A region=f7a538750ba0f201503066ae75fe1e58] 2024-12-09T23:59:36,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/101b6a16ccd94c279d658f083a6e7be4 is 175, key is test_row_0/A:col10/1733788775374/Put/seqid=0 2024-12-09T23:59:36,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742339_1515 (size=30955) 2024-12-09T23:59:36,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:36,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:36,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:36,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788836533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:36,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:36,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:36,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:36,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788836534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:36,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:36,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:36,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:36,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:36,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:36,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788836536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:36,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:36,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:36,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:36,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:36,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788836536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:36,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:36,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:36,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:36,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-09T23:59:36,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:36,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788836643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:36,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:36,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:36,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:36,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:36,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788836643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:36,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:36,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:36,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:36,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:36,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788836643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:36,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:36,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:36,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788836643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:36,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:36,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:36,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:36,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:36,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:36,825 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=113, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/101b6a16ccd94c279d658f083a6e7be4 2024-12-09T23:59:36,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/9553276f6d934d10b37ba941bada43ec is 50, key is test_row_0/B:col10/1733788775374/Put/seqid=0 2024-12-09T23:59:36,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742340_1516 (size=12001) 2024-12-09T23:59:36,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:36,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788836852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:36,855 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:36,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:36,855 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:36,855 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:36,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788836853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:36,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:36,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:36,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:36,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:36,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788836853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:36,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:36,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:36,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:36,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:36,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788836853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:36,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:36,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:36,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:36,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-09T23:59:37,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:37,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788837163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788837163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:37,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:37,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788837163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788837164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:37,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:37,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:37,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:37,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:37,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:37,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:37,234 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/9553276f6d934d10b37ba941bada43ec 2024-12-09T23:59:37,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/6744dce079a14d19aaff2e678bbb1df7 is 50, key is test_row_0/C:col10/1733788775374/Put/seqid=0 2024-12-09T23:59:37,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742341_1517 (size=12001) 2024-12-09T23:59:37,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-09T23:59:37,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:37,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40458 deadline: 1733788837403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:37,404 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_0' in f7a538750ba0f201503066ae75fe1e58 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:59:37,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:37,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:37,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:37,646 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/6744dce079a14d19aaff2e678bbb1df7 2024-12-09T23:59:37,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/101b6a16ccd94c279d658f083a6e7be4 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/101b6a16ccd94c279d658f083a6e7be4 2024-12-09T23:59:37,651 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/101b6a16ccd94c279d658f083a6e7be4, entries=150, sequenceid=113, filesize=30.2 K 2024-12-09T23:59:37,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/9553276f6d934d10b37ba941bada43ec as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/9553276f6d934d10b37ba941bada43ec 2024-12-09T23:59:37,655 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/9553276f6d934d10b37ba941bada43ec, entries=150, sequenceid=113, filesize=11.7 K 2024-12-09T23:59:37,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/6744dce079a14d19aaff2e678bbb1df7 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/6744dce079a14d19aaff2e678bbb1df7 2024-12-09T23:59:37,658 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/6744dce079a14d19aaff2e678bbb1df7, entries=150, sequenceid=113, filesize=11.7 K 2024-12-09T23:59:37,659 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f7a538750ba0f201503066ae75fe1e58 in 1252ms, sequenceid=113, compaction requested=true 2024-12-09T23:59:37,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for f7a538750ba0f201503066ae75fe1e58: 2024-12-09T23:59:37,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:37,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-12-09T23:59:37,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-12-09T23:59:37,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-09T23:59:37,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4040 sec 2024-12-09T23:59:37,663 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.4080 sec 2024-12-09T23:59:37,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:37,678 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7a538750ba0f201503066ae75fe1e58 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T23:59:37,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=A 2024-12-09T23:59:37,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:37,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=B 2024-12-09T23:59:37,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:37,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=C 2024-12-09T23:59:37,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:37,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099ff045d7ed464bdb93a0cdacde7ee57f_f7a538750ba0f201503066ae75fe1e58 is 50, key is test_row_0/A:col10/1733788777677/Put/seqid=0 2024-12-09T23:59:37,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742342_1518 (size=14744) 2024-12-09T23:59:37,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:37,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788837688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:37,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:37,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:37,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:37,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:37,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788837689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:37,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:37,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:37,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:37,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:37,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788837690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:37,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:37,697 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:37,697 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:37,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788837691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:37,697 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:37,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:37,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:37,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:37,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:37,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788837803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:37,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:37,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:37,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:37,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:37,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788837803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:37,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:37,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:37,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:37,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788837803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:37,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:37,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:37,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:37,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:37,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788837803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:37,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:37,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:37,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:37,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:38,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:38,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788838013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:38,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:38,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:38,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:38,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:38,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788838013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:38,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:38,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:38,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:38,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:38,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788838013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:38,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:38,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:38,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:38,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:38,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788838013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:38,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:38,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:38,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:38,086 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:38,089 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099ff045d7ed464bdb93a0cdacde7ee57f_f7a538750ba0f201503066ae75fe1e58 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099ff045d7ed464bdb93a0cdacde7ee57f_f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:38,090 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/6475e4a26df14cf083e21d12996be875, store: [table=TestAcidGuarantees family=A region=f7a538750ba0f201503066ae75fe1e58] 2024-12-09T23:59:38,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/6475e4a26df14cf083e21d12996be875 is 175, key is test_row_0/A:col10/1733788777677/Put/seqid=0 2024-12-09T23:59:38,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742343_1519 (size=39699) 2024-12-09T23:59:38,293 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T23:59:38,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:38,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788838323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:38,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:38,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:38,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:38,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:38,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:38,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788838323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:38,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788838323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:38,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:38,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:38,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:38,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:38,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:38,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:38,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:38,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788838323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:38,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:38,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:38,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:38,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-09T23:59:38,403 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:59:38,403 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:59:38,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-12-09T23:59:38,405 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:59:38,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T23:59:38,405 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:59:38,405 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:59:38,493 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/6475e4a26df14cf083e21d12996be875 2024-12-09T23:59:38,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/19b00ef690aa4f58abf9004cb29a939a is 50, key is test_row_0/B:col10/1733788777677/Put/seqid=0 2024-12-09T23:59:38,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742344_1520 (size=12101) 2024-12-09T23:59:38,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T23:59:38,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-09T23:59:38,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:38,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:38,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:38,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:38,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:38,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:38,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-09T23:59:38,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:38,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:38,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:38,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:38,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:38,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:38,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T23:59:38,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:38,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788838833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:38,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:38,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:38,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788838833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:38,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:38,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:38,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:38,836 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:38,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788838833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:38,836 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:38,836 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:38,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:38,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:38,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:38,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788838833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:38,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:38,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:38,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:38,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:38,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-09T23:59:38,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:38,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:38,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:38,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:38,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:38,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:38,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/19b00ef690aa4f58abf9004cb29a939a 2024-12-09T23:59:38,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/d5dd0193e7ae45399a6584235c7f3c03 is 50, key is test_row_0/C:col10/1733788777677/Put/seqid=0 2024-12-09T23:59:38,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742345_1521 (size=12101) 2024-12-09T23:59:39,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-09T23:59:39,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:39,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:39,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:39,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:39,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:39,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:39,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T23:59:39,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-09T23:59:39,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:39,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:39,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:39,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:39,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:39,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:39,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/d5dd0193e7ae45399a6584235c7f3c03 2024-12-09T23:59:39,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/6475e4a26df14cf083e21d12996be875 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/6475e4a26df14cf083e21d12996be875 2024-12-09T23:59:39,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/6475e4a26df14cf083e21d12996be875, entries=200, sequenceid=133, filesize=38.8 K 2024-12-09T23:59:39,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/19b00ef690aa4f58abf9004cb29a939a as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/19b00ef690aa4f58abf9004cb29a939a 2024-12-09T23:59:39,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/19b00ef690aa4f58abf9004cb29a939a, entries=150, sequenceid=133, filesize=11.8 K 2024-12-09T23:59:39,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/d5dd0193e7ae45399a6584235c7f3c03 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/d5dd0193e7ae45399a6584235c7f3c03 2024-12-09T23:59:39,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/d5dd0193e7ae45399a6584235c7f3c03, entries=150, sequenceid=133, filesize=11.8 K 2024-12-09T23:59:39,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for f7a538750ba0f201503066ae75fe1e58 in 1647ms, sequenceid=133, compaction requested=true 2024-12-09T23:59:39,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7a538750ba0f201503066ae75fe1e58: 2024-12-09T23:59:39,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7a538750ba0f201503066ae75fe1e58:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:59:39,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:59:39,325 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 8 store files, 0 compacting, 8 eligible, 16 blocking 2024-12-09T23:59:39,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7a538750ba0f201503066ae75fe1e58:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:59:39,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:59:39,325 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 8 store files, 0 compacting, 8 eligible, 16 blocking 2024-12-09T23:59:39,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7a538750ba0f201503066ae75fe1e58:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:59:39,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:59:39,327 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 8 files of size 96108 starting at candidate #0 after considering 21 permutations with 21 in ratio 2024-12-09T23:59:39,327 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 8 files of size 256384 starting at candidate #0 after considering 21 permutations with 21 in ratio 2024-12-09T23:59:39,327 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): f7a538750ba0f201503066ae75fe1e58/B is initiating minor compaction (all files) 2024-12-09T23:59:39,327 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): f7a538750ba0f201503066ae75fe1e58/A is initiating minor compaction (all files) 2024-12-09T23:59:39,327 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7a538750ba0f201503066ae75fe1e58/A in TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:39,327 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7a538750ba0f201503066ae75fe1e58/B in TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:39,327 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/cdb04b7b249347668bdb3199534c0728, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/b1348fbd47ae4af8a42c5e451985d121, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/8d994d3917f941f28358d4e495c9da79, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/77ff084e8b274127aba6844500da0c52, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/e0aaafe4fc1a4cfdb9ff2c6b9acb4ffc, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/9168cd5a74294c24b56416abce10a5d3, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/101b6a16ccd94c279d658f083a6e7be4, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/6475e4a26df14cf083e21d12996be875] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp, totalSize=250.4 K 2024-12-09T23:59:39,327 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/7e364e022c9b4ff2a23d3a153bc76833, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/9a5ce0fc8d4a4abba199e8fcb9ecb685, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/db3e6e74422b485fbd370937be3585dd, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/abb6b565b4f444c4b1bd65b299ec7ab5, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/7a3667c0c3224003bef6949a53c4ad19, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/9f4e399827d64e79b1f697d3f310e0b4, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/9553276f6d934d10b37ba941bada43ec, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/19b00ef690aa4f58abf9004cb29a939a] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp, totalSize=93.9 K 2024-12-09T23:59:39,327 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=8 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:39,327 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. files: [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/cdb04b7b249347668bdb3199534c0728, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/b1348fbd47ae4af8a42c5e451985d121, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/8d994d3917f941f28358d4e495c9da79, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/77ff084e8b274127aba6844500da0c52, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/e0aaafe4fc1a4cfdb9ff2c6b9acb4ffc, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/9168cd5a74294c24b56416abce10a5d3, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/101b6a16ccd94c279d658f083a6e7be4, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/6475e4a26df14cf083e21d12996be875] 2024-12-09T23:59:39,327 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e364e022c9b4ff2a23d3a153bc76833, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733788764511 2024-12-09T23:59:39,328 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdb04b7b249347668bdb3199534c0728, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733788764511 2024-12-09T23:59:39,328 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a5ce0fc8d4a4abba199e8fcb9ecb685, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733788764546 2024-12-09T23:59:39,328 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1348fbd47ae4af8a42c5e451985d121, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733788764546 2024-12-09T23:59:39,328 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting db3e6e74422b485fbd370937be3585dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733788766705 2024-12-09T23:59:39,328 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d994d3917f941f28358d4e495c9da79, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733788766705 2024-12-09T23:59:39,328 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting abb6b565b4f444c4b1bd65b299ec7ab5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733788768877 2024-12-09T23:59:39,328 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77ff084e8b274127aba6844500da0c52, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733788768877 2024-12-09T23:59:39,328 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0aaafe4fc1a4cfdb9ff2c6b9acb4ffc, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733788771039 2024-12-09T23:59:39,328 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a3667c0c3224003bef6949a53c4ad19, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733788771039 2024-12-09T23:59:39,329 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9168cd5a74294c24b56416abce10a5d3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733788773211 2024-12-09T23:59:39,329 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f4e399827d64e79b1f697d3f310e0b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733788773211 2024-12-09T23:59:39,329 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 101b6a16ccd94c279d658f083a6e7be4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1733788775366 2024-12-09T23:59:39,329 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 9553276f6d934d10b37ba941bada43ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1733788775366 2024-12-09T23:59:39,329 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6475e4a26df14cf083e21d12996be875, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733788776526 2024-12-09T23:59:39,329 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 19b00ef690aa4f58abf9004cb29a939a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733788776526 2024-12-09T23:59:39,336 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f7a538750ba0f201503066ae75fe1e58] 2024-12-09T23:59:39,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-09T23:59:39,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:39,338 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing f7a538750ba0f201503066ae75fe1e58 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T23:59:39,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=A 2024-12-09T23:59:39,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:39,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=B 2024-12-09T23:59:39,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:39,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=C 2024-12-09T23:59:39,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:39,339 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7a538750ba0f201503066ae75fe1e58#B#compaction#445 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:59:39,340 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/efd3ff65c43049e895c836af445c0217 is 50, key is test_row_0/B:col10/1733788777677/Put/seqid=0 2024-12-09T23:59:39,341 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120971986507c7bf4c8c80614093d106b250_f7a538750ba0f201503066ae75fe1e58 store=[table=TestAcidGuarantees family=A region=f7a538750ba0f201503066ae75fe1e58] 2024-12-09T23:59:39,346 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120971986507c7bf4c8c80614093d106b250_f7a538750ba0f201503066ae75fe1e58, store=[table=TestAcidGuarantees family=A region=f7a538750ba0f201503066ae75fe1e58] 2024-12-09T23:59:39,346 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120971986507c7bf4c8c80614093d106b250_f7a538750ba0f201503066ae75fe1e58 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f7a538750ba0f201503066ae75fe1e58] 2024-12-09T23:59:39,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412091e14f960b556455aab35eae3e3cc9e48_f7a538750ba0f201503066ae75fe1e58 is 50, key is test_row_0/A:col10/1733788777690/Put/seqid=0 2024-12-09T23:59:39,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742346_1522 (size=12375) 2024-12-09T23:59:39,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742347_1523 (size=4469) 2024-12-09T23:59:39,357 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7a538750ba0f201503066ae75fe1e58#A#compaction#444 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:59:39,358 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/20b51409f3ec4646985cb6aab219b773 is 175, key is test_row_0/A:col10/1733788777677/Put/seqid=0 2024-12-09T23:59:39,360 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/efd3ff65c43049e895c836af445c0217 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/efd3ff65c43049e895c836af445c0217 2024-12-09T23:59:39,363 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 8 (all) file(s) in f7a538750ba0f201503066ae75fe1e58/B of f7a538750ba0f201503066ae75fe1e58 into efd3ff65c43049e895c836af445c0217(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:59:39,363 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7a538750ba0f201503066ae75fe1e58: 2024-12-09T23:59:39,363 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., storeName=f7a538750ba0f201503066ae75fe1e58/B, priority=8, startTime=1733788779325; duration=0sec 2024-12-09T23:59:39,363 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:59:39,363 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7a538750ba0f201503066ae75fe1e58:B 2024-12-09T23:59:39,363 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 8 store files, 0 compacting, 8 eligible, 16 blocking 2024-12-09T23:59:39,365 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 8 files of size 96108 starting at candidate #0 after considering 21 permutations with 21 in ratio 2024-12-09T23:59:39,365 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): f7a538750ba0f201503066ae75fe1e58/C is initiating minor compaction (all files) 2024-12-09T23:59:39,365 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7a538750ba0f201503066ae75fe1e58/C in TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:39,365 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/8db7d472b0ae429dbfa4145d10d7c5b3, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/b6074e701b4c45e4ab177e43b66dddf1, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/b5c386ca3be94452ab94548b45eb9f4f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/ab6a8e90cb2d4247ad32fbe488d632f4, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/c2b03ca56f6b4eebaa27320fef402aa4, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/e046d140922c40a9a20255cccd2cac30, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/6744dce079a14d19aaff2e678bbb1df7, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/d5dd0193e7ae45399a6584235c7f3c03] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp, totalSize=93.9 K 2024-12-09T23:59:39,365 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 8db7d472b0ae429dbfa4145d10d7c5b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733788764511 2024-12-09T23:59:39,365 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting b6074e701b4c45e4ab177e43b66dddf1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733788764546 2024-12-09T23:59:39,366 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting b5c386ca3be94452ab94548b45eb9f4f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733788766705 2024-12-09T23:59:39,366 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting ab6a8e90cb2d4247ad32fbe488d632f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733788768877 2024-12-09T23:59:39,366 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting c2b03ca56f6b4eebaa27320fef402aa4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733788771039 2024-12-09T23:59:39,366 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting e046d140922c40a9a20255cccd2cac30, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733788773211 2024-12-09T23:59:39,366 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 6744dce079a14d19aaff2e678bbb1df7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1733788775366 2024-12-09T23:59:39,367 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting d5dd0193e7ae45399a6584235c7f3c03, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733788776526 2024-12-09T23:59:39,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742349_1525 (size=31329) 2024-12-09T23:59:39,380 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7a538750ba0f201503066ae75fe1e58#C#compaction#447 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:59:39,381 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/c6837db7bb8d47689c16c6ddf6111774 is 50, key is test_row_0/C:col10/1733788777677/Put/seqid=0 2024-12-09T23:59:39,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742348_1524 (size=12304) 2024-12-09T23:59:39,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:39,385 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/20b51409f3ec4646985cb6aab219b773 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/20b51409f3ec4646985cb6aab219b773 2024-12-09T23:59:39,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742350_1526 (size=12375) 2024-12-09T23:59:39,389 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412091e14f960b556455aab35eae3e3cc9e48_f7a538750ba0f201503066ae75fe1e58 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091e14f960b556455aab35eae3e3cc9e48_f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:39,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/4f800a1f75434a6caf60f16a00c36baa, store: [table=TestAcidGuarantees family=A region=f7a538750ba0f201503066ae75fe1e58] 2024-12-09T23:59:39,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/4f800a1f75434a6caf60f16a00c36baa is 175, key is test_row_0/A:col10/1733788777690/Put/seqid=0 2024-12-09T23:59:39,391 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 8 (all) file(s) in f7a538750ba0f201503066ae75fe1e58/A of f7a538750ba0f201503066ae75fe1e58 into 20b51409f3ec4646985cb6aab219b773(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:59:39,391 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7a538750ba0f201503066ae75fe1e58: 2024-12-09T23:59:39,391 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., storeName=f7a538750ba0f201503066ae75fe1e58/A, priority=8, startTime=1733788779325; duration=0sec 2024-12-09T23:59:39,391 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:59:39,391 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7a538750ba0f201503066ae75fe1e58:A 2024-12-09T23:59:39,394 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/c6837db7bb8d47689c16c6ddf6111774 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/c6837db7bb8d47689c16c6ddf6111774 2024-12-09T23:59:39,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742351_1527 (size=31105) 2024-12-09T23:59:39,397 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 8 (all) file(s) in f7a538750ba0f201503066ae75fe1e58/C of f7a538750ba0f201503066ae75fe1e58 into c6837db7bb8d47689c16c6ddf6111774(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:59:39,397 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7a538750ba0f201503066ae75fe1e58: 2024-12-09T23:59:39,397 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., storeName=f7a538750ba0f201503066ae75fe1e58/C, priority=8, startTime=1733788779325; duration=0sec 2024-12-09T23:59:39,397 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:59:39,398 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7a538750ba0f201503066ae75fe1e58:C 2024-12-09T23:59:39,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T23:59:39,795 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/4f800a1f75434a6caf60f16a00c36baa 2024-12-09T23:59:39,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/cb65809730b1406da451f4a1feda7e21 is 50, key is test_row_0/B:col10/1733788777690/Put/seqid=0 2024-12-09T23:59:39,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742352_1528 (size=12151) 2024-12-09T23:59:39,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:39,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:39,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:39,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788839862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:39,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:39,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:39,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788839862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:39,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788839863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:39,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:39,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788839865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:39,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:39,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:39,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:39,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:39,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788839973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:39,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:39,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:39,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788839973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:39,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:39,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:39,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:39,975 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:39,975 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:39,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788839973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:39,975 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:39,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:39,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:39,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788839973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:39,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:39,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:39,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:39,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:39,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:40,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788840183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:40,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788840183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:40,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:40,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:40,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788840183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:40,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:40,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788840183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:40,211 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/cb65809730b1406da451f4a1feda7e21 2024-12-09T23:59:40,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/06a7232b9fd84623b59f593f276b91af is 50, key is test_row_0/C:col10/1733788777690/Put/seqid=0 2024-12-09T23:59:40,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742353_1529 (size=12151) 2024-12-09T23:59:40,220 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/06a7232b9fd84623b59f593f276b91af 2024-12-09T23:59:40,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/4f800a1f75434a6caf60f16a00c36baa as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/4f800a1f75434a6caf60f16a00c36baa 2024-12-09T23:59:40,227 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/4f800a1f75434a6caf60f16a00c36baa, entries=150, sequenceid=145, filesize=30.4 K 2024-12-09T23:59:40,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/cb65809730b1406da451f4a1feda7e21 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/cb65809730b1406da451f4a1feda7e21 2024-12-09T23:59:40,232 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/cb65809730b1406da451f4a1feda7e21, entries=150, sequenceid=145, filesize=11.9 K 2024-12-09T23:59:40,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/06a7232b9fd84623b59f593f276b91af as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/06a7232b9fd84623b59f593f276b91af 2024-12-09T23:59:40,235 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/06a7232b9fd84623b59f593f276b91af, entries=150, sequenceid=145, filesize=11.9 K 2024-12-09T23:59:40,236 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f7a538750ba0f201503066ae75fe1e58 in 898ms, sequenceid=145, compaction requested=false 2024-12-09T23:59:40,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for f7a538750ba0f201503066ae75fe1e58: 2024-12-09T23:59:40,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:40,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-12-09T23:59:40,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-12-09T23:59:40,239 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-09T23:59:40,239 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8320 sec 2024-12-09T23:59:40,240 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.8360 sec 2024-12-09T23:59:40,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:40,494 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7a538750ba0f201503066ae75fe1e58 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-09T23:59:40,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=A 2024-12-09T23:59:40,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:59:40,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=B 2024-12-09T23:59:40,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:59:40,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=C 2024-12-09T23:59:40,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:59:40,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209db60cff400d64e8babed1522ce4fa3d3_f7a538750ba0f201503066ae75fe1e58 is 50, key is test_row_0/A:col10/1733788780493/Put/seqid=0 2024-12-09T23:59:40,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742354_1530 (size=14794) 2024-12-09T23:59:40,503 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:40,506 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209db60cff400d64e8babed1522ce4fa3d3_f7a538750ba0f201503066ae75fe1e58 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209db60cff400d64e8babed1522ce4fa3d3_f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:40,506 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/a91e705a5015487e837da03990e59e0d, store: [table=TestAcidGuarantees family=A region=f7a538750ba0f201503066ae75fe1e58] 2024-12-09T23:59:40,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:40,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788840501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:40,507 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:40,507 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:40,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:40,507 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:40,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/a91e705a5015487e837da03990e59e0d is 175, key is test_row_0/A:col10/1733788780493/Put/seqid=0 2024-12-09T23:59:40,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788840501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:40,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:40,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:40,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:40,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:40,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788840502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:40,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:40,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:40,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:40,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:40,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788840503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:40,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:40,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:40,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:40,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742355_1531 (size=39749) 2024-12-09T23:59:40,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T23:59:40,553 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:59:40,553 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:59:40,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-12-09T23:59:40,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-09T23:59:40,555 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:59:40,555 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:59:40,555 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:59:40,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:40,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788840613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:40,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:40,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:40,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:40,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788840613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:40,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:40,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:40,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:40,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:40,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:40,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788840613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:40,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:40,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:40,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788840613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:40,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:40,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:40,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:40,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:40,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:40,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-09T23:59:40,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-09T23:59:40,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:40,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:40,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:40,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:40,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:40,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:40,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:40,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788840823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:40,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:40,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:40,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:40,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:40,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788840823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:40,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:40,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:40,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788840823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:40,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:40,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:40,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:40,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:40,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:40,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:40,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788840823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:40,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:40,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:40,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:40,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-09T23:59:40,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:40,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:40,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:40,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:40,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:40,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:40,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-09T23:59:40,910 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/a91e705a5015487e837da03990e59e0d 2024-12-09T23:59:40,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/83caeb4af6df49018fbf465192135ecd is 50, key is test_row_0/B:col10/1733788780493/Put/seqid=0 2024-12-09T23:59:40,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742356_1532 (size=12151) 2024-12-09T23:59:41,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-09T23:59:41,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:41,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:41,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:41,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:41,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:41,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:41,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:41,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788841133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:41,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:41,135 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:41,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788841133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:41,135 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:41,135 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:41,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:41,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:41,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:41,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:41,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788841133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:41,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:41,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:41,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:41,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:41,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788841133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:41,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:41,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:41,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:41,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-09T23:59:41,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:41,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:41,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:41,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:41,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:41,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:41,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-09T23:59:41,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-09T23:59:41,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:41,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:41,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:41,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:41,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:41,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/83caeb4af6df49018fbf465192135ecd 2024-12-09T23:59:41,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:41,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/c95cb38ebd0b4425a636e1dd8a18f884 is 50, key is test_row_0/C:col10/1733788780493/Put/seqid=0 2024-12-09T23:59:41,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742357_1533 (size=12151) 2024-12-09T23:59:41,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:41,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40458 deadline: 1733788841422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:41,424 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 2907d75fbb3e,37627,1733788599837 for 'test_row_0' in f7a538750ba0f201503066ae75fe1e58 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T23:59:41,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:41,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:41,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:41,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-09T23:59:41,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:41,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:41,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:41,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:41,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:41,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:41,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-09T23:59:41,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:41,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:41,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:41,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:41,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:41,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:41,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:41,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788841643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:41,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:41,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:41,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:41,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788841643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:41,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:41,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:41,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:41,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:41,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:41,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788841643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:41,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:41,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:41,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788841643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:41,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:41,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:41,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:41,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:41,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:41,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-09T23:59:41,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/c95cb38ebd0b4425a636e1dd8a18f884 2024-12-09T23:59:41,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/a91e705a5015487e837da03990e59e0d as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/a91e705a5015487e837da03990e59e0d 2024-12-09T23:59:41,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/a91e705a5015487e837da03990e59e0d, entries=200, sequenceid=169, filesize=38.8 K 2024-12-09T23:59:41,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/83caeb4af6df49018fbf465192135ecd as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/83caeb4af6df49018fbf465192135ecd 2024-12-09T23:59:41,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/83caeb4af6df49018fbf465192135ecd, entries=150, sequenceid=169, filesize=11.9 K 2024-12-09T23:59:41,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/c95cb38ebd0b4425a636e1dd8a18f884 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/c95cb38ebd0b4425a636e1dd8a18f884 2024-12-09T23:59:41,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/c95cb38ebd0b4425a636e1dd8a18f884, entries=150, sequenceid=169, filesize=11.9 K 2024-12-09T23:59:41,739 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for f7a538750ba0f201503066ae75fe1e58 in 1245ms, sequenceid=169, compaction requested=true 2024-12-09T23:59:41,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7a538750ba0f201503066ae75fe1e58: 2024-12-09T23:59:41,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7a538750ba0f201503066ae75fe1e58:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:59:41,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:59:41,739 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:59:41,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7a538750ba0f201503066ae75fe1e58:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:59:41,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:59:41,739 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:59:41,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7a538750ba0f201503066ae75fe1e58:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:59:41,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:59:41,740 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:59:41,740 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:59:41,740 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): f7a538750ba0f201503066ae75fe1e58/A is initiating minor compaction (all files) 2024-12-09T23:59:41,740 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): f7a538750ba0f201503066ae75fe1e58/B is initiating minor compaction (all files) 2024-12-09T23:59:41,740 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7a538750ba0f201503066ae75fe1e58/B in TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:41,740 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7a538750ba0f201503066ae75fe1e58/A in TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:41,740 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/20b51409f3ec4646985cb6aab219b773, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/4f800a1f75434a6caf60f16a00c36baa, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/a91e705a5015487e837da03990e59e0d] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp, totalSize=99.8 K 2024-12-09T23:59:41,740 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/efd3ff65c43049e895c836af445c0217, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/cb65809730b1406da451f4a1feda7e21, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/83caeb4af6df49018fbf465192135ecd] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp, totalSize=35.8 K 2024-12-09T23:59:41,740 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:41,740 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. files: [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/20b51409f3ec4646985cb6aab219b773, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/4f800a1f75434a6caf60f16a00c36baa, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/a91e705a5015487e837da03990e59e0d] 2024-12-09T23:59:41,740 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting efd3ff65c43049e895c836af445c0217, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733788776526 2024-12-09T23:59:41,741 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20b51409f3ec4646985cb6aab219b773, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733788776526 2024-12-09T23:59:41,741 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting cb65809730b1406da451f4a1feda7e21, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733788777678 2024-12-09T23:59:41,741 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 83caeb4af6df49018fbf465192135ecd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733788779861 2024-12-09T23:59:41,741 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f800a1f75434a6caf60f16a00c36baa, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733788777678 2024-12-09T23:59:41,741 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting a91e705a5015487e837da03990e59e0d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733788779861 2024-12-09T23:59:41,750 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f7a538750ba0f201503066ae75fe1e58] 2024-12-09T23:59:41,751 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209cf5d08fdb07d413da01d37bc5cc7df02_f7a538750ba0f201503066ae75fe1e58 store=[table=TestAcidGuarantees family=A region=f7a538750ba0f201503066ae75fe1e58] 2024-12-09T23:59:41,751 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7a538750ba0f201503066ae75fe1e58#B#compaction#453 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:59:41,752 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/4076a4b2d8624c9ba84c2bc81132b1db is 50, key is test_row_0/B:col10/1733788780493/Put/seqid=0 2024-12-09T23:59:41,753 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209cf5d08fdb07d413da01d37bc5cc7df02_f7a538750ba0f201503066ae75fe1e58, store=[table=TestAcidGuarantees family=A region=f7a538750ba0f201503066ae75fe1e58] 2024-12-09T23:59:41,753 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209cf5d08fdb07d413da01d37bc5cc7df02_f7a538750ba0f201503066ae75fe1e58 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f7a538750ba0f201503066ae75fe1e58] 2024-12-09T23:59:41,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742358_1534 (size=12527) 2024-12-09T23:59:41,762 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/4076a4b2d8624c9ba84c2bc81132b1db as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/4076a4b2d8624c9ba84c2bc81132b1db 2024-12-09T23:59:41,766 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7a538750ba0f201503066ae75fe1e58/B of f7a538750ba0f201503066ae75fe1e58 into 4076a4b2d8624c9ba84c2bc81132b1db(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:59:41,766 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7a538750ba0f201503066ae75fe1e58: 2024-12-09T23:59:41,766 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., storeName=f7a538750ba0f201503066ae75fe1e58/B, priority=13, startTime=1733788781739; duration=0sec 2024-12-09T23:59:41,766 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:59:41,766 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7a538750ba0f201503066ae75fe1e58:B 2024-12-09T23:59:41,766 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:59:41,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742359_1535 (size=4469) 2024-12-09T23:59:41,767 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:59:41,767 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): f7a538750ba0f201503066ae75fe1e58/C is initiating minor compaction (all files) 2024-12-09T23:59:41,767 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7a538750ba0f201503066ae75fe1e58/C in TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:41,768 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/c6837db7bb8d47689c16c6ddf6111774, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/06a7232b9fd84623b59f593f276b91af, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/c95cb38ebd0b4425a636e1dd8a18f884] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp, totalSize=35.8 K 2024-12-09T23:59:41,768 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting c6837db7bb8d47689c16c6ddf6111774, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733788776526 2024-12-09T23:59:41,768 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 06a7232b9fd84623b59f593f276b91af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733788777678 2024-12-09T23:59:41,768 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7a538750ba0f201503066ae75fe1e58#A#compaction#454 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:59:41,768 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting c95cb38ebd0b4425a636e1dd8a18f884, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733788779861 2024-12-09T23:59:41,768 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/184fbe3e4a914940bb1cf1862cd76c75 is 175, key is test_row_0/A:col10/1733788780493/Put/seqid=0 2024-12-09T23:59:41,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742360_1536 (size=31481) 2024-12-09T23:59:41,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-09T23:59:41,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:41,774 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7a538750ba0f201503066ae75fe1e58#C#compaction#455 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:59:41,774 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing f7a538750ba0f201503066ae75fe1e58 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T23:59:41,774 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/385da97c047c4939a33f21e6f3c16c71 is 50, key is test_row_0/C:col10/1733788780493/Put/seqid=0 2024-12-09T23:59:41,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=A 2024-12-09T23:59:41,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:41,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=B 2024-12-09T23:59:41,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:41,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=C 2024-12-09T23:59:41,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:41,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742361_1537 (size=12527) 2024-12-09T23:59:41,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412096290ea46d2d24953a934e052fd4ee8d4_f7a538750ba0f201503066ae75fe1e58 is 50, key is test_row_0/A:col10/1733788780501/Put/seqid=0 2024-12-09T23:59:41,783 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/385da97c047c4939a33f21e6f3c16c71 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/385da97c047c4939a33f21e6f3c16c71 2024-12-09T23:59:41,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742362_1538 (size=12304) 2024-12-09T23:59:41,787 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7a538750ba0f201503066ae75fe1e58/C of f7a538750ba0f201503066ae75fe1e58 into 385da97c047c4939a33f21e6f3c16c71(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:59:41,787 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7a538750ba0f201503066ae75fe1e58: 2024-12-09T23:59:41,787 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., storeName=f7a538750ba0f201503066ae75fe1e58/C, priority=13, startTime=1733788781739; duration=0sec 2024-12-09T23:59:41,787 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:59:41,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:41,787 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7a538750ba0f201503066ae75fe1e58:C 2024-12-09T23:59:41,790 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412096290ea46d2d24953a934e052fd4ee8d4_f7a538750ba0f201503066ae75fe1e58 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096290ea46d2d24953a934e052fd4ee8d4_f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:41,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/f2255a43cdd544fd8be3d0028f6d8dde, store: [table=TestAcidGuarantees family=A region=f7a538750ba0f201503066ae75fe1e58] 2024-12-09T23:59:41,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/f2255a43cdd544fd8be3d0028f6d8dde is 175, key is test_row_0/A:col10/1733788780501/Put/seqid=0 2024-12-09T23:59:41,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742363_1539 (size=31105) 2024-12-09T23:59:42,175 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/184fbe3e4a914940bb1cf1862cd76c75 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/184fbe3e4a914940bb1cf1862cd76c75 2024-12-09T23:59:42,179 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7a538750ba0f201503066ae75fe1e58/A of f7a538750ba0f201503066ae75fe1e58 into 184fbe3e4a914940bb1cf1862cd76c75(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:59:42,179 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7a538750ba0f201503066ae75fe1e58: 2024-12-09T23:59:42,179 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., storeName=f7a538750ba0f201503066ae75fe1e58/A, priority=13, startTime=1733788781739; duration=0sec 2024-12-09T23:59:42,179 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:59:42,179 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7a538750ba0f201503066ae75fe1e58:A 2024-12-09T23:59:42,196 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=181, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/f2255a43cdd544fd8be3d0028f6d8dde 2024-12-09T23:59:42,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/6ee5237684e942118f9cb5aa5d5ea689 is 50, key is test_row_0/B:col10/1733788780501/Put/seqid=0 2024-12-09T23:59:42,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742364_1540 (size=12151) 2024-12-09T23:59:42,204 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/6ee5237684e942118f9cb5aa5d5ea689 2024-12-09T23:59:42,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/54110162cd204adb90c9a7c483a38e86 is 50, key is test_row_0/C:col10/1733788780501/Put/seqid=0 2024-12-09T23:59:42,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742365_1541 (size=12151) 2024-12-09T23:59:42,625 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/54110162cd204adb90c9a7c483a38e86 2024-12-09T23:59:42,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/f2255a43cdd544fd8be3d0028f6d8dde as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/f2255a43cdd544fd8be3d0028f6d8dde 2024-12-09T23:59:42,631 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/f2255a43cdd544fd8be3d0028f6d8dde, entries=150, sequenceid=181, filesize=30.4 K 2024-12-09T23:59:42,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/6ee5237684e942118f9cb5aa5d5ea689 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/6ee5237684e942118f9cb5aa5d5ea689 2024-12-09T23:59:42,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,635 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/6ee5237684e942118f9cb5aa5d5ea689, entries=150, sequenceid=181, filesize=11.9 K 2024-12-09T23:59:42,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/54110162cd204adb90c9a7c483a38e86 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/54110162cd204adb90c9a7c483a38e86 2024-12-09T23:59:42,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,639 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/54110162cd204adb90c9a7c483a38e86, entries=150, sequenceid=181, filesize=11.9 K 2024-12-09T23:59:42,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,640 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for f7a538750ba0f201503066ae75fe1e58 in 865ms, sequenceid=181, compaction requested=false 2024-12-09T23:59:42,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for f7a538750ba0f201503066ae75fe1e58: 2024-12-09T23:59:42,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:42,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-12-09T23:59:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-12-09T23:59:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,642 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-12-09T23:59:42,642 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0860 sec 2024-12-09T23:59:42,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,644 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 2.0900 sec 2024-12-09T23:59:42,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:42,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7a538750ba0f201503066ae75fe1e58 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T23:59:42,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=A 2024-12-09T23:59:42,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:42,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=B 2024-12-09T23:59:42,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:42,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=C 2024-12-09T23:59:42,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:42,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209a9f97fe8d0d54a1dadd67fc5923117e1_f7a538750ba0f201503066ae75fe1e58 is 50, key is test_row_0/A:col10/1733788782681/Put/seqid=0 2024-12-09T23:59:42,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742366_1542 (size=22268) 2024-12-09T23:59:42,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:42,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-09T23:59:42,703 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:59:42,704 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T23:59:42,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-12-09T23:59:42,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-09T23:59:42,705 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T23:59:42,706 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T23:59:42,706 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T23:59:42,710 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733788782698, max timestamp=1733788782708Num uniques 859; 2024-12-09T23:59:42,710 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733788782698, max timestamp=1733788782708Num uniques 859; 2024-12-09T23:59:42,710 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733788782698, max timestamp=1733788782708Num uniques 859; 2024-12-09T23:59:42,710 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T23:59:42,710 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T23:59:42,710 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2907d75fbb3e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T23:59:42,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:42,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788842724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:42,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:42,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:42,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:42,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:42,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788842728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:42,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:42,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788842729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:42,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:42,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:42,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:42,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788842730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:42,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:42,734 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:42,734 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:42,734 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:42,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-09T23:59:42,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:42,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788842833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:42,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:42,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:42,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:42,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:42,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788842842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:42,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:42,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:42,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:42,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:42,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788842842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:42,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:42,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:42,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:42,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788842842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:42,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:42,845 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:42,845 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:42,845 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:42,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-09T23:59:42,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:42,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:42,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:42,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:42,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:42,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:43,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-09T23:59:43,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:43,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:43,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:43,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:43,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:43,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:43,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-09T23:59:43,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:43,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788843042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:43,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:43,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:43,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:43,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:43,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788843052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:43,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:43,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:43,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:43,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788843053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:43,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:43,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:43,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:43,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:43,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788843053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:43,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:43,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:43,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:43,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:43,097 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:43,100 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209a9f97fe8d0d54a1dadd67fc5923117e1_f7a538750ba0f201503066ae75fe1e58 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a9f97fe8d0d54a1dadd67fc5923117e1_f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:43,100 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/9c1e18aa70ce4c63b01cbed784573ee5, store: [table=TestAcidGuarantees family=A region=f7a538750ba0f201503066ae75fe1e58] 2024-12-09T23:59:43,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/9c1e18aa70ce4c63b01cbed784573ee5 is 175, key is test_row_0/A:col10/1733788782681/Put/seqid=0 2024-12-09T23:59:43,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742367_1543 (size=65673) 2024-12-09T23:59:43,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-09T23:59:43,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:43,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:43,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:43,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:43,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:43,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:43,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-09T23:59:43,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:43,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:43,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:43,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:43,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:43,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:43,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-09T23:59:43,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:43,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788843353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:43,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:43,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:43,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:43,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:43,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788843362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:43,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:43,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:43,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788843362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:43,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788843362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:43,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-09T23:59:43,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:43,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:43,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:43,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:43,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:43,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:43,511 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=193, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/9c1e18aa70ce4c63b01cbed784573ee5 2024-12-09T23:59:43,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/c7bcb07ffc32402182fff796fc1b33b2 is 50, key is test_row_0/B:col10/1733788782681/Put/seqid=0 2024-12-09T23:59:43,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742368_1544 (size=12151) 2024-12-09T23:59:43,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/c7bcb07ffc32402182fff796fc1b33b2 2024-12-09T23:59:43,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/d61e4e41ae194cfb86293bc350396c7b is 50, key is test_row_0/C:col10/1733788782681/Put/seqid=0 2024-12-09T23:59:43,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742369_1545 (size=12151) 2024-12-09T23:59:43,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-09T23:59:43,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:43,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:43,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:43,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:43,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:43,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:43,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-09T23:59:43,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:43,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:43,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:43,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:43,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:43,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:43,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-09T23:59:43,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:43,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40424 deadline: 1733788843862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:43,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:43,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:43,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:43,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:43,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40398 deadline: 1733788843872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:43,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:43,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:43,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:43,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:43,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40400 deadline: 1733788843873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:43,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T23:59:43,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:43,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37627 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1733788843873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:43,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:43,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:43,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 , the old value is region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:43,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T23:59:43,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., hostname=2907d75fbb3e,37627,1733788599837, seqNum=5 because the exception is null or not the one we care about 2024-12-09T23:59:43,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-09T23:59:43,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:43,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:43,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:43,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:43,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:43,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:59:43,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/d61e4e41ae194cfb86293bc350396c7b 2024-12-09T23:59:43,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/9c1e18aa70ce4c63b01cbed784573ee5 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/9c1e18aa70ce4c63b01cbed784573ee5 2024-12-09T23:59:43,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/9c1e18aa70ce4c63b01cbed784573ee5, entries=350, sequenceid=193, filesize=64.1 K 2024-12-09T23:59:43,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/c7bcb07ffc32402182fff796fc1b33b2 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/c7bcb07ffc32402182fff796fc1b33b2 2024-12-09T23:59:43,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/c7bcb07ffc32402182fff796fc1b33b2, entries=150, sequenceid=193, filesize=11.9 K 2024-12-09T23:59:43,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/d61e4e41ae194cfb86293bc350396c7b as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/d61e4e41ae194cfb86293bc350396c7b 2024-12-09T23:59:43,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/d61e4e41ae194cfb86293bc350396c7b, entries=150, sequenceid=193, filesize=11.9 K 2024-12-09T23:59:43,939 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f7a538750ba0f201503066ae75fe1e58 in 1255ms, sequenceid=193, compaction requested=true 2024-12-09T23:59:43,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7a538750ba0f201503066ae75fe1e58: 2024-12-09T23:59:43,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7a538750ba0f201503066ae75fe1e58:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T23:59:43,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:59:43,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7a538750ba0f201503066ae75fe1e58:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T23:59:43,940 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:59:43,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:59:43,940 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:59:43,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7a538750ba0f201503066ae75fe1e58:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T23:59:43,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:59:43,940 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:59:43,940 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:59:43,940 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1541): f7a538750ba0f201503066ae75fe1e58/A is initiating minor compaction (all files) 2024-12-09T23:59:43,940 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): f7a538750ba0f201503066ae75fe1e58/B is initiating minor compaction (all files) 2024-12-09T23:59:43,941 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7a538750ba0f201503066ae75fe1e58/A in TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:43,941 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7a538750ba0f201503066ae75fe1e58/B in TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:43,941 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/184fbe3e4a914940bb1cf1862cd76c75, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/f2255a43cdd544fd8be3d0028f6d8dde, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/9c1e18aa70ce4c63b01cbed784573ee5] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp, totalSize=125.3 K 2024-12-09T23:59:43,941 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/4076a4b2d8624c9ba84c2bc81132b1db, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/6ee5237684e942118f9cb5aa5d5ea689, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/c7bcb07ffc32402182fff796fc1b33b2] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp, totalSize=36.0 K 2024-12-09T23:59:43,941 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:43,941 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. files: [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/184fbe3e4a914940bb1cf1862cd76c75, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/f2255a43cdd544fd8be3d0028f6d8dde, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/9c1e18aa70ce4c63b01cbed784573ee5] 2024-12-09T23:59:43,941 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 4076a4b2d8624c9ba84c2bc81132b1db, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733788779861 2024-12-09T23:59:43,941 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 184fbe3e4a914940bb1cf1862cd76c75, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733788779861 2024-12-09T23:59:43,941 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ee5237684e942118f9cb5aa5d5ea689, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1733788780499 2024-12-09T23:59:43,941 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2255a43cdd544fd8be3d0028f6d8dde, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1733788780499 2024-12-09T23:59:43,941 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting c7bcb07ffc32402182fff796fc1b33b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733788782671 2024-12-09T23:59:43,942 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c1e18aa70ce4c63b01cbed784573ee5, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733788782670 2024-12-09T23:59:43,945 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f7a538750ba0f201503066ae75fe1e58] 2024-12-09T23:59:43,946 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7a538750ba0f201503066ae75fe1e58#B#compaction#462 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:59:43,947 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/9e6acd13e0a04a96a0e43418b7e7053d is 50, key is test_row_0/B:col10/1733788782681/Put/seqid=0 2024-12-09T23:59:43,948 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209d8cb174207d34a9b9e3504c71e811bae_f7a538750ba0f201503066ae75fe1e58 store=[table=TestAcidGuarantees family=A region=f7a538750ba0f201503066ae75fe1e58] 2024-12-09T23:59:43,950 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209d8cb174207d34a9b9e3504c71e811bae_f7a538750ba0f201503066ae75fe1e58, store=[table=TestAcidGuarantees family=A region=f7a538750ba0f201503066ae75fe1e58] 2024-12-09T23:59:43,950 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d8cb174207d34a9b9e3504c71e811bae_f7a538750ba0f201503066ae75fe1e58 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f7a538750ba0f201503066ae75fe1e58] 2024-12-09T23:59:43,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742370_1546 (size=12629) 2024-12-09T23:59:43,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742371_1547 (size=4469) 2024-12-09T23:59:44,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-09T23:59:44,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:44,077 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing f7a538750ba0f201503066ae75fe1e58 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-09T23:59:44,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=A 2024-12-09T23:59:44,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:59:44,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=B 2024-12-09T23:59:44,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:59:44,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=C 2024-12-09T23:59:44,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T23:59:44,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209a75c69a27d7a43e2bf867f3cedd49f9f_f7a538750ba0f201503066ae75fe1e58 is 50, key is test_row_0/A:col10/1733788782722/Put/seqid=0 2024-12-09T23:59:44,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742372_1548 (size=12304) 2024-12-09T23:59:44,354 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7a538750ba0f201503066ae75fe1e58#A#compaction#463 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:59:44,355 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/d5449cbb5ca84b2797e61a98d0d71089 is 175, key is test_row_0/A:col10/1733788782681/Put/seqid=0 2024-12-09T23:59:44,356 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/9e6acd13e0a04a96a0e43418b7e7053d as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/9e6acd13e0a04a96a0e43418b7e7053d 2024-12-09T23:59:44,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742373_1549 (size=31583) 2024-12-09T23:59:44,361 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7a538750ba0f201503066ae75fe1e58/B of f7a538750ba0f201503066ae75fe1e58 into 9e6acd13e0a04a96a0e43418b7e7053d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:59:44,361 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7a538750ba0f201503066ae75fe1e58: 2024-12-09T23:59:44,361 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., storeName=f7a538750ba0f201503066ae75fe1e58/B, priority=13, startTime=1733788783940; duration=0sec 2024-12-09T23:59:44,361 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T23:59:44,361 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7a538750ba0f201503066ae75fe1e58:B 2024-12-09T23:59:44,361 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T23:59:44,362 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/d5449cbb5ca84b2797e61a98d0d71089 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/d5449cbb5ca84b2797e61a98d0d71089 2024-12-09T23:59:44,362 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T23:59:44,362 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1541): f7a538750ba0f201503066ae75fe1e58/C is initiating minor compaction (all files) 2024-12-09T23:59:44,362 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7a538750ba0f201503066ae75fe1e58/C in TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:44,362 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/385da97c047c4939a33f21e6f3c16c71, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/54110162cd204adb90c9a7c483a38e86, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/d61e4e41ae194cfb86293bc350396c7b] into tmpdir=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp, totalSize=36.0 K 2024-12-09T23:59:44,362 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 385da97c047c4939a33f21e6f3c16c71, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733788779861 2024-12-09T23:59:44,363 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting 54110162cd204adb90c9a7c483a38e86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1733788780499 2024-12-09T23:59:44,363 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] compactions.Compactor(225): Compacting d61e4e41ae194cfb86293bc350396c7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733788782671 2024-12-09T23:59:44,366 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7a538750ba0f201503066ae75fe1e58/A of f7a538750ba0f201503066ae75fe1e58 into d5449cbb5ca84b2797e61a98d0d71089(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:59:44,366 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7a538750ba0f201503066ae75fe1e58: 2024-12-09T23:59:44,366 INFO [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., storeName=f7a538750ba0f201503066ae75fe1e58/A, priority=13, startTime=1733788783940; duration=0sec 2024-12-09T23:59:44,366 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:59:44,366 DEBUG [RS:0;2907d75fbb3e:37627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7a538750ba0f201503066ae75fe1e58:A 2024-12-09T23:59:44,368 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7a538750ba0f201503066ae75fe1e58#C#compaction#465 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T23:59:44,368 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/b772bf89dc4447cdb072ec84cac5d653 is 50, key is test_row_0/C:col10/1733788782681/Put/seqid=0 2024-12-09T23:59:44,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742374_1550 (size=12629) 2024-12-09T23:59:44,503 INFO [Thread-2218 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2218. 2024-12-09T23:59:44,503 DEBUG [Thread-2218 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:59:44,503 DEBUG [Thread-2218 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:44,503 DEBUG [Thread-2218 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:44,503 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:59:44,503 INFO [Thread-2224 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2224. 2024-12-09T23:59:44,503 DEBUG [Thread-2224 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:59:44,503 DEBUG [Thread-2224 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:44,503 DEBUG [Thread-2224 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:44,504 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:59:44,504 INFO [Thread-2226 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2226. 2024-12-09T23:59:44,504 DEBUG [Thread-2226 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:59:44,504 DEBUG [Thread-2226 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:44,504 DEBUG [Thread-2226 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:44,504 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:59:44,505 INFO [Thread-2220 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2220. 2024-12-09T23:59:44,505 DEBUG [Thread-2220 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:59:44,505 DEBUG [Thread-2220 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:44,505 DEBUG [Thread-2220 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:44,505 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:59:44,505 INFO [Thread-2222 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2222. 2024-12-09T23:59:44,505 DEBUG [Thread-2222 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:59:44,505 DEBUG [Thread-2222 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:44,505 DEBUG [Thread-2222 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:44,505 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:59:44,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:44,541 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209a75c69a27d7a43e2bf867f3cedd49f9f_f7a538750ba0f201503066ae75fe1e58 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a75c69a27d7a43e2bf867f3cedd49f9f_f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:44,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/8204b5b63877428e91f96a090bd089f0, store: [table=TestAcidGuarantees family=A region=f7a538750ba0f201503066ae75fe1e58] 2024-12-09T23:59:44,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/8204b5b63877428e91f96a090bd089f0 is 175, key is test_row_0/A:col10/1733788782722/Put/seqid=0 2024-12-09T23:59:44,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742375_1551 (size=31105) 2024-12-09T23:59:44,784 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/b772bf89dc4447cdb072ec84cac5d653 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/b772bf89dc4447cdb072ec84cac5d653 2024-12-09T23:59:44,789 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7a538750ba0f201503066ae75fe1e58/C of f7a538750ba0f201503066ae75fe1e58 into b772bf89dc4447cdb072ec84cac5d653(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T23:59:44,789 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7a538750ba0f201503066ae75fe1e58: 2024-12-09T23:59:44,789 INFO [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., storeName=f7a538750ba0f201503066ae75fe1e58/C, priority=13, startTime=1733788783940; duration=0sec 2024-12-09T23:59:44,789 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T23:59:44,789 DEBUG [RS:0;2907d75fbb3e:37627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7a538750ba0f201503066ae75fe1e58:C 2024-12-09T23:59:44,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-09T23:59:44,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37627 {}] regionserver.HRegion(8855): Flush requested on f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:44,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. as already flushing 2024-12-09T23:59:44,888 INFO [Thread-2207 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2207. 2024-12-09T23:59:44,888 INFO [Thread-2215 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2215. 2024-12-09T23:59:44,888 DEBUG [Thread-2207 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:59:44,888 INFO [Thread-2209 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2209. 2024-12-09T23:59:44,888 DEBUG [Thread-2215 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:59:44,888 DEBUG [Thread-2207 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:44,888 DEBUG [Thread-2215 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:44,888 DEBUG [Thread-2209 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:59:44,888 DEBUG [Thread-2209 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:44,888 DEBUG [Thread-2207 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:44,888 DEBUG [Thread-2215 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:44,888 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:59:44,888 DEBUG [Thread-2209 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:44,888 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:59:44,888 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:59:44,894 INFO [Thread-2211 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2211. 2024-12-09T23:59:44,894 DEBUG [Thread-2211 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:59:44,894 DEBUG [Thread-2211 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:44,894 DEBUG [Thread-2211 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:44,894 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:59:44,949 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=228, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/8204b5b63877428e91f96a090bd089f0 2024-12-09T23:59:44,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/aacdb65b02284f4ebd43ffc6d3226b56 is 50, key is test_row_0/B:col10/1733788782722/Put/seqid=0 2024-12-09T23:59:44,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742376_1552 (size=12151) 2024-12-09T23:59:45,368 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/aacdb65b02284f4ebd43ffc6d3226b56 2024-12-09T23:59:45,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/559bbe81583946909072c0c0e15721a3 is 50, key is test_row_0/C:col10/1733788782722/Put/seqid=0 2024-12-09T23:59:45,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742377_1553 (size=12151) 2024-12-09T23:59:45,787 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/559bbe81583946909072c0c0e15721a3 2024-12-09T23:59:45,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/8204b5b63877428e91f96a090bd089f0 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/8204b5b63877428e91f96a090bd089f0 2024-12-09T23:59:45,804 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/8204b5b63877428e91f96a090bd089f0, entries=150, sequenceid=228, filesize=30.4 K 2024-12-09T23:59:45,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/aacdb65b02284f4ebd43ffc6d3226b56 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/aacdb65b02284f4ebd43ffc6d3226b56 2024-12-09T23:59:45,809 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/aacdb65b02284f4ebd43ffc6d3226b56, entries=150, sequenceid=228, filesize=11.9 K 2024-12-09T23:59:45,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/559bbe81583946909072c0c0e15721a3 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/559bbe81583946909072c0c0e15721a3 2024-12-09T23:59:45,814 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/559bbe81583946909072c0c0e15721a3, entries=150, sequenceid=228, filesize=11.9 K 2024-12-09T23:59:45,814 INFO [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=26.84 KB/27480 for f7a538750ba0f201503066ae75fe1e58 in 1738ms, sequenceid=228, compaction requested=false 2024-12-09T23:59:45,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for f7a538750ba0f201503066ae75fe1e58: 2024-12-09T23:59:45,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:45,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2907d75fbb3e:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-12-09T23:59:45,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-12-09T23:59:45,817 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-12-09T23:59:45,817 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1100 sec 2024-12-09T23:59:45,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 3.1140 sec 2024-12-09T23:59:46,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-09T23:59:46,874 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:59:51,518 INFO [Thread-2213 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2213. 2024-12-09T23:59:51,518 DEBUG [Thread-2213 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T23:59:51,518 DEBUG [Thread-2213 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:51,518 DEBUG [Thread-2213 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:51,518 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:59:51,519 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T23:59:51,519 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-12-09T23:59:51,519 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-12-09T23:59:51,519 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-12-09T23:59:51,519 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-12-09T23:59:51,519 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-12-09T23:59:51,519 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T23:59:51,519 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4410 2024-12-09T23:59:51,519 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4411 2024-12-09T23:59:51,519 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4617 2024-12-09T23:59:51,519 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4402 2024-12-09T23:59:51,519 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4413 2024-12-09T23:59:51,519 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T23:59:51,520 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T23:59:51,520 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T23:59:51,520 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:51,521 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:51,521 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:59:51,522 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-09T23:59:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T23:59:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-09T23:59:51,528 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733788791527"}]},"ts":"1733788791527"} 2024-12-09T23:59:51,529 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T23:59:51,530 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T23:59:51,530 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T23:59:51,533 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7a538750ba0f201503066ae75fe1e58, UNASSIGN}] 2024-12-09T23:59:51,534 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7a538750ba0f201503066ae75fe1e58, UNASSIGN 2024-12-09T23:59:51,535 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=f7a538750ba0f201503066ae75fe1e58, regionState=CLOSING, regionLocation=2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:51,537 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7a538750ba0f201503066ae75fe1e58, UNASSIGN because future has completed 2024-12-09T23:59:51,537 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T23:59:51,537 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837}] 2024-12-09T23:59:51,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-09T23:59:51,689 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(122): Close f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:51,689 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T23:59:51,690 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1722): Closing f7a538750ba0f201503066ae75fe1e58, disabling compactions & flushes 2024-12-09T23:59:51,690 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:51,690 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:51,690 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. after waiting 0 ms 2024-12-09T23:59:51,690 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:51,690 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(2902): Flushing f7a538750ba0f201503066ae75fe1e58 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-09T23:59:51,691 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=A 2024-12-09T23:59:51,691 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:51,691 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=B 2024-12-09T23:59:51,691 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:51,691 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f7a538750ba0f201503066ae75fe1e58, store=C 2024-12-09T23:59:51,691 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T23:59:51,703 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209b01a48b9df0f4cb0ac8b24294707e0dd_f7a538750ba0f201503066ae75fe1e58 is 50, key is test_row_0/A:col10/1733788791514/Put/seqid=0 2024-12-09T23:59:51,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742378_1554 (size=12304) 2024-12-09T23:59:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-09T23:59:52,108 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:59:52,117 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209b01a48b9df0f4cb0ac8b24294707e0dd_f7a538750ba0f201503066ae75fe1e58 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b01a48b9df0f4cb0ac8b24294707e0dd_f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:52,118 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/0e39c9ff5e8d485da7593e7844e14187, store: [table=TestAcidGuarantees family=A region=f7a538750ba0f201503066ae75fe1e58] 2024-12-09T23:59:52,118 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/0e39c9ff5e8d485da7593e7844e14187 is 175, key is test_row_0/A:col10/1733788791514/Put/seqid=0 2024-12-09T23:59:52,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742379_1555 (size=31105) 2024-12-09T23:59:52,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-09T23:59:52,524 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=239, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/0e39c9ff5e8d485da7593e7844e14187 2024-12-09T23:59:52,535 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/43b83fd882344e36816c16434b45e27f is 50, key is test_row_0/B:col10/1733788791514/Put/seqid=0 2024-12-09T23:59:52,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742380_1556 (size=12151) 2024-12-09T23:59:52,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-09T23:59:52,941 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/43b83fd882344e36816c16434b45e27f 2024-12-09T23:59:52,954 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/3d2297692d53438faf045960df8b934f is 50, key is test_row_0/C:col10/1733788791514/Put/seqid=0 2024-12-09T23:59:52,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742381_1557 (size=12151) 2024-12-09T23:59:53,359 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/3d2297692d53438faf045960df8b934f 2024-12-09T23:59:53,370 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/A/0e39c9ff5e8d485da7593e7844e14187 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/0e39c9ff5e8d485da7593e7844e14187 2024-12-09T23:59:53,374 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/0e39c9ff5e8d485da7593e7844e14187, entries=150, sequenceid=239, filesize=30.4 K 2024-12-09T23:59:53,375 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/B/43b83fd882344e36816c16434b45e27f as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/43b83fd882344e36816c16434b45e27f 2024-12-09T23:59:53,378 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/43b83fd882344e36816c16434b45e27f, entries=150, sequenceid=239, filesize=11.9 K 2024-12-09T23:59:53,378 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/.tmp/C/3d2297692d53438faf045960df8b934f as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/3d2297692d53438faf045960df8b934f 2024-12-09T23:59:53,381 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/3d2297692d53438faf045960df8b934f, entries=150, sequenceid=239, filesize=11.9 K 2024-12-09T23:59:53,382 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f7a538750ba0f201503066ae75fe1e58 in 1692ms, sequenceid=239, compaction requested=true 2024-12-09T23:59:53,382 DEBUG [StoreCloser-TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/cdb04b7b249347668bdb3199534c0728, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/b1348fbd47ae4af8a42c5e451985d121, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/8d994d3917f941f28358d4e495c9da79, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/77ff084e8b274127aba6844500da0c52, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/e0aaafe4fc1a4cfdb9ff2c6b9acb4ffc, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/9168cd5a74294c24b56416abce10a5d3, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/101b6a16ccd94c279d658f083a6e7be4, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/6475e4a26df14cf083e21d12996be875, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/20b51409f3ec4646985cb6aab219b773, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/4f800a1f75434a6caf60f16a00c36baa, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/a91e705a5015487e837da03990e59e0d, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/184fbe3e4a914940bb1cf1862cd76c75, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/f2255a43cdd544fd8be3d0028f6d8dde, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/9c1e18aa70ce4c63b01cbed784573ee5] to archive 2024-12-09T23:59:53,383 DEBUG [StoreCloser-TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-09T23:59:53,385 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/cdb04b7b249347668bdb3199534c0728 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/cdb04b7b249347668bdb3199534c0728 2024-12-09T23:59:53,386 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/b1348fbd47ae4af8a42c5e451985d121 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/b1348fbd47ae4af8a42c5e451985d121 2024-12-09T23:59:53,386 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/77ff084e8b274127aba6844500da0c52 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/77ff084e8b274127aba6844500da0c52 2024-12-09T23:59:53,387 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/8d994d3917f941f28358d4e495c9da79 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/8d994d3917f941f28358d4e495c9da79 2024-12-09T23:59:53,387 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/e0aaafe4fc1a4cfdb9ff2c6b9acb4ffc to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/e0aaafe4fc1a4cfdb9ff2c6b9acb4ffc 2024-12-09T23:59:53,387 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/101b6a16ccd94c279d658f083a6e7be4 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/101b6a16ccd94c279d658f083a6e7be4 2024-12-09T23:59:53,387 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/9168cd5a74294c24b56416abce10a5d3 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/9168cd5a74294c24b56416abce10a5d3 2024-12-09T23:59:53,387 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/6475e4a26df14cf083e21d12996be875 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/6475e4a26df14cf083e21d12996be875 2024-12-09T23:59:53,387 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/20b51409f3ec4646985cb6aab219b773 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/20b51409f3ec4646985cb6aab219b773 2024-12-09T23:59:53,388 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/4f800a1f75434a6caf60f16a00c36baa to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/4f800a1f75434a6caf60f16a00c36baa 2024-12-09T23:59:53,388 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/a91e705a5015487e837da03990e59e0d to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/a91e705a5015487e837da03990e59e0d 2024-12-09T23:59:53,388 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/9c1e18aa70ce4c63b01cbed784573ee5 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/9c1e18aa70ce4c63b01cbed784573ee5 2024-12-09T23:59:53,388 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/184fbe3e4a914940bb1cf1862cd76c75 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/184fbe3e4a914940bb1cf1862cd76c75 2024-12-09T23:59:53,389 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/f2255a43cdd544fd8be3d0028f6d8dde to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/f2255a43cdd544fd8be3d0028f6d8dde 2024-12-09T23:59:53,390 DEBUG [StoreCloser-TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/7e364e022c9b4ff2a23d3a153bc76833, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/9a5ce0fc8d4a4abba199e8fcb9ecb685, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/db3e6e74422b485fbd370937be3585dd, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/abb6b565b4f444c4b1bd65b299ec7ab5, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/7a3667c0c3224003bef6949a53c4ad19, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/9f4e399827d64e79b1f697d3f310e0b4, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/9553276f6d934d10b37ba941bada43ec, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/efd3ff65c43049e895c836af445c0217, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/19b00ef690aa4f58abf9004cb29a939a, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/cb65809730b1406da451f4a1feda7e21, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/4076a4b2d8624c9ba84c2bc81132b1db, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/83caeb4af6df49018fbf465192135ecd, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/6ee5237684e942118f9cb5aa5d5ea689, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/c7bcb07ffc32402182fff796fc1b33b2] to archive 2024-12-09T23:59:53,390 DEBUG [StoreCloser-TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-09T23:59:53,392 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/abb6b565b4f444c4b1bd65b299ec7ab5 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/abb6b565b4f444c4b1bd65b299ec7ab5 2024-12-09T23:59:53,392 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/efd3ff65c43049e895c836af445c0217 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/efd3ff65c43049e895c836af445c0217 2024-12-09T23:59:53,392 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/db3e6e74422b485fbd370937be3585dd to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/db3e6e74422b485fbd370937be3585dd 2024-12-09T23:59:53,392 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/9a5ce0fc8d4a4abba199e8fcb9ecb685 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/9a5ce0fc8d4a4abba199e8fcb9ecb685 2024-12-09T23:59:53,392 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/7a3667c0c3224003bef6949a53c4ad19 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/7a3667c0c3224003bef6949a53c4ad19 2024-12-09T23:59:53,392 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/9f4e399827d64e79b1f697d3f310e0b4 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/9f4e399827d64e79b1f697d3f310e0b4 2024-12-09T23:59:53,392 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/7e364e022c9b4ff2a23d3a153bc76833 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/7e364e022c9b4ff2a23d3a153bc76833 2024-12-09T23:59:53,393 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/9553276f6d934d10b37ba941bada43ec to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/9553276f6d934d10b37ba941bada43ec 2024-12-09T23:59:53,393 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/cb65809730b1406da451f4a1feda7e21 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/cb65809730b1406da451f4a1feda7e21 2024-12-09T23:59:53,393 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/4076a4b2d8624c9ba84c2bc81132b1db to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/4076a4b2d8624c9ba84c2bc81132b1db 2024-12-09T23:59:53,393 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/19b00ef690aa4f58abf9004cb29a939a to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/19b00ef690aa4f58abf9004cb29a939a 2024-12-09T23:59:53,393 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/c7bcb07ffc32402182fff796fc1b33b2 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/c7bcb07ffc32402182fff796fc1b33b2 2024-12-09T23:59:53,393 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/6ee5237684e942118f9cb5aa5d5ea689 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/6ee5237684e942118f9cb5aa5d5ea689 2024-12-09T23:59:53,393 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/83caeb4af6df49018fbf465192135ecd to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/83caeb4af6df49018fbf465192135ecd 2024-12-09T23:59:53,394 DEBUG [StoreCloser-TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/8db7d472b0ae429dbfa4145d10d7c5b3, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/b6074e701b4c45e4ab177e43b66dddf1, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/b5c386ca3be94452ab94548b45eb9f4f, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/ab6a8e90cb2d4247ad32fbe488d632f4, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/c2b03ca56f6b4eebaa27320fef402aa4, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/e046d140922c40a9a20255cccd2cac30, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/6744dce079a14d19aaff2e678bbb1df7, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/c6837db7bb8d47689c16c6ddf6111774, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/d5dd0193e7ae45399a6584235c7f3c03, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/06a7232b9fd84623b59f593f276b91af, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/385da97c047c4939a33f21e6f3c16c71, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/c95cb38ebd0b4425a636e1dd8a18f884, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/54110162cd204adb90c9a7c483a38e86, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/d61e4e41ae194cfb86293bc350396c7b] to archive 2024-12-09T23:59:53,395 DEBUG [StoreCloser-TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-09T23:59:53,396 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/8db7d472b0ae429dbfa4145d10d7c5b3 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/8db7d472b0ae429dbfa4145d10d7c5b3 2024-12-09T23:59:53,396 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/e046d140922c40a9a20255cccd2cac30 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/e046d140922c40a9a20255cccd2cac30 2024-12-09T23:59:53,396 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/ab6a8e90cb2d4247ad32fbe488d632f4 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/ab6a8e90cb2d4247ad32fbe488d632f4 2024-12-09T23:59:53,396 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/b5c386ca3be94452ab94548b45eb9f4f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/b5c386ca3be94452ab94548b45eb9f4f 2024-12-09T23:59:53,397 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/b6074e701b4c45e4ab177e43b66dddf1 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/b6074e701b4c45e4ab177e43b66dddf1 2024-12-09T23:59:53,397 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/6744dce079a14d19aaff2e678bbb1df7 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/6744dce079a14d19aaff2e678bbb1df7 2024-12-09T23:59:53,397 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/c6837db7bb8d47689c16c6ddf6111774 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/c6837db7bb8d47689c16c6ddf6111774 2024-12-09T23:59:53,397 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/c2b03ca56f6b4eebaa27320fef402aa4 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/c2b03ca56f6b4eebaa27320fef402aa4 2024-12-09T23:59:53,397 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/d5dd0193e7ae45399a6584235c7f3c03 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/d5dd0193e7ae45399a6584235c7f3c03 2024-12-09T23:59:53,398 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/06a7232b9fd84623b59f593f276b91af to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/06a7232b9fd84623b59f593f276b91af 2024-12-09T23:59:53,398 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/c95cb38ebd0b4425a636e1dd8a18f884 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/c95cb38ebd0b4425a636e1dd8a18f884 2024-12-09T23:59:53,398 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/385da97c047c4939a33f21e6f3c16c71 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/385da97c047c4939a33f21e6f3c16c71 2024-12-09T23:59:53,398 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/54110162cd204adb90c9a7c483a38e86 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/54110162cd204adb90c9a7c483a38e86 2024-12-09T23:59:53,398 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/d61e4e41ae194cfb86293bc350396c7b to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/d61e4e41ae194cfb86293bc350396c7b 2024-12-09T23:59:53,401 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/recovered.edits/242.seqid, newMaxSeqId=242, maxSeqId=4 2024-12-09T23:59:53,401 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. 2024-12-09T23:59:53,401 DEBUG [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1676): Region close journal for f7a538750ba0f201503066ae75fe1e58: Waiting for close lock at 1733788791690Running coprocessor pre-close hooks at 1733788791690Disabling compacts and flushes for region at 1733788791690Disabling writes for close at 1733788791690Obtaining lock to block concurrent updates at 1733788791690Preparing flush snapshotting stores in f7a538750ba0f201503066ae75fe1e58 at 1733788791690Finished memstore snapshotting TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733788791691 (+1 ms)Flushing stores of TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58. at 1733788791692 (+1 ms)Flushing f7a538750ba0f201503066ae75fe1e58/A: creating writer at 1733788791692Flushing mob file f7a538750ba0f201503066ae75fe1e58/A: appending metadata at 1733788791703 (+11 ms)Flushing mob file f7a538750ba0f201503066ae75fe1e58/A: closing flushed file at 1733788791703Flushing f7a538750ba0f201503066ae75fe1e58/A: appending metadata at 1733788792118 (+415 ms)Flushing f7a538750ba0f201503066ae75fe1e58/A: closing flushed file at 1733788792118Flushing f7a538750ba0f201503066ae75fe1e58/B: creating writer at 1733788792531 (+413 ms)Flushing f7a538750ba0f201503066ae75fe1e58/B: appending metadata at 1733788792535 (+4 ms)Flushing f7a538750ba0f201503066ae75fe1e58/B: closing flushed file at 1733788792535Flushing f7a538750ba0f201503066ae75fe1e58/C: creating writer at 1733788792951 (+416 ms)Flushing f7a538750ba0f201503066ae75fe1e58/C: appending metadata at 1733788792953 (+2 ms)Flushing f7a538750ba0f201503066ae75fe1e58/C: closing flushed file at 1733788792953Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3671e825: reopening flushed file at 1733788793368 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e3235a: reopening flushed file at 1733788793374 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6659aea: reopening flushed file at 1733788793378 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f7a538750ba0f201503066ae75fe1e58 in 1692ms, sequenceid=239, compaction requested=true at 1733788793382 (+4 ms)Writing region close event to WAL at 1733788793399 (+17 ms)Running coprocessor post-close hooks at 1733788793401 (+2 ms)Closed at 1733788793401 2024-12-09T23:59:53,403 INFO [RS_CLOSE_REGION-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(157): Closed f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:53,403 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=f7a538750ba0f201503066ae75fe1e58, regionState=CLOSED 2024-12-09T23:59:53,404 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 because future has completed 2024-12-09T23:59:53,406 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-12-09T23:59:53,406 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseRegionProcedure f7a538750ba0f201503066ae75fe1e58, server=2907d75fbb3e,37627,1733788599837 in 1.8680 sec 2024-12-09T23:59:53,408 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=170, resume processing ppid=169 2024-12-09T23:59:53,408 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, ppid=169, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7a538750ba0f201503066ae75fe1e58, UNASSIGN in 1.8740 sec 2024-12-09T23:59:53,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-12-09T23:59:53,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8780 sec 2024-12-09T23:59:53,410 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733788793410"}]},"ts":"1733788793410"} 2024-12-09T23:59:53,411 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T23:59:53,411 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T23:59:53,413 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8890 sec 2024-12-09T23:59:53,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-09T23:59:53,685 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:59:53,686 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-09T23:59:53,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T23:59:53,691 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T23:59:53,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-09T23:59:53,694 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=172, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T23:59:53,699 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:53,702 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A, FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B, FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C, FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/recovered.edits] 2024-12-09T23:59:53,706 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/0e39c9ff5e8d485da7593e7844e14187 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/0e39c9ff5e8d485da7593e7844e14187 2024-12-09T23:59:53,706 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/8204b5b63877428e91f96a090bd089f0 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/8204b5b63877428e91f96a090bd089f0 2024-12-09T23:59:53,706 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/d5449cbb5ca84b2797e61a98d0d71089 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/A/d5449cbb5ca84b2797e61a98d0d71089 2024-12-09T23:59:53,710 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/43b83fd882344e36816c16434b45e27f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/43b83fd882344e36816c16434b45e27f 2024-12-09T23:59:53,710 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/aacdb65b02284f4ebd43ffc6d3226b56 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/aacdb65b02284f4ebd43ffc6d3226b56 2024-12-09T23:59:53,710 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/9e6acd13e0a04a96a0e43418b7e7053d to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/B/9e6acd13e0a04a96a0e43418b7e7053d 2024-12-09T23:59:53,714 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/3d2297692d53438faf045960df8b934f to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/3d2297692d53438faf045960df8b934f 2024-12-09T23:59:53,714 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/559bbe81583946909072c0c0e15721a3 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/559bbe81583946909072c0c0e15721a3 2024-12-09T23:59:53,714 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/b772bf89dc4447cdb072ec84cac5d653 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/C/b772bf89dc4447cdb072ec84cac5d653 2024-12-09T23:59:53,718 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/recovered.edits/242.seqid to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58/recovered.edits/242.seqid 2024-12-09T23:59:53,718 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/default/TestAcidGuarantees/f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:53,719 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T23:59:53,719 DEBUG [PEWorker-5 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T23:59:53,720 DEBUG [PEWorker-5 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-09T23:59:53,729 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091b2e5008e1ca45e5b7ffddbdd14f506c_f7a538750ba0f201503066ae75fe1e58 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091b2e5008e1ca45e5b7ffddbdd14f506c_f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:53,729 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091e14f960b556455aab35eae3e3cc9e48_f7a538750ba0f201503066ae75fe1e58 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091e14f960b556455aab35eae3e3cc9e48_f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:53,729 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120924d87fa53b924c1d80d174b7eb801eac_f7a538750ba0f201503066ae75fe1e58 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120924d87fa53b924c1d80d174b7eb801eac_f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:53,729 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099ba197c7d2164053909619d57a6ea6ff_f7a538750ba0f201503066ae75fe1e58 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099ba197c7d2164053909619d57a6ea6ff_f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:53,729 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096290ea46d2d24953a934e052fd4ee8d4_f7a538750ba0f201503066ae75fe1e58 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096290ea46d2d24953a934e052fd4ee8d4_f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:53,729 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a75c69a27d7a43e2bf867f3cedd49f9f_f7a538750ba0f201503066ae75fe1e58 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a75c69a27d7a43e2bf867f3cedd49f9f_f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:53,730 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099ff045d7ed464bdb93a0cdacde7ee57f_f7a538750ba0f201503066ae75fe1e58 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099ff045d7ed464bdb93a0cdacde7ee57f_f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:53,731 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a9f97fe8d0d54a1dadd67fc5923117e1_f7a538750ba0f201503066ae75fe1e58 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a9f97fe8d0d54a1dadd67fc5923117e1_f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:53,731 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b01a48b9df0f4cb0ac8b24294707e0dd_f7a538750ba0f201503066ae75fe1e58 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b01a48b9df0f4cb0ac8b24294707e0dd_f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:53,731 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209db304e03ce8540d2a0f090daa3d9135d_f7a538750ba0f201503066ae75fe1e58 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209db304e03ce8540d2a0f090daa3d9135d_f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:53,731 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209db60cff400d64e8babed1522ce4fa3d3_f7a538750ba0f201503066ae75fe1e58 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209db60cff400d64e8babed1522ce4fa3d3_f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:53,731 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209adce10c870ce48a98d1dc85198ae90f8_f7a538750ba0f201503066ae75fe1e58 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209adce10c870ce48a98d1dc85198ae90f8_f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:53,731 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c60f4e7fa81f4c75857d11e7c695a6cc_f7a538750ba0f201503066ae75fe1e58 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c60f4e7fa81f4c75857d11e7c695a6cc_f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:53,736 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099945818e2a45425d84541785f4348dee_f7a538750ba0f201503066ae75fe1e58 to hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099945818e2a45425d84541785f4348dee_f7a538750ba0f201503066ae75fe1e58 2024-12-09T23:59:53,737 DEBUG [PEWorker-5 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T23:59:53,739 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=172, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T23:59:53,741 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T23:59:53,742 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T23:59:53,743 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=172, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T23:59:53,743 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-09T23:59:53,743 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733788793743"}]},"ts":"9223372036854775807"} 2024-12-09T23:59:53,745 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T23:59:53,745 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f7a538750ba0f201503066ae75fe1e58, NAME => 'TestAcidGuarantees,,1733788761347.f7a538750ba0f201503066ae75fe1e58.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T23:59:53,745 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T23:59:53,745 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733788793745"}]},"ts":"9223372036854775807"} 2024-12-09T23:59:53,746 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T23:59:53,746 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=172, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T23:59:53,747 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 60 msec 2024-12-09T23:59:53,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-09T23:59:53,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T23:59:53,803 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T23:59:53,818 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=255 (was 254) - Thread LEAK? -, OpenFileDescriptor=449 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=285 (was 256) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3980 (was 3992) 2024-12-09T23:59:53,818 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-09T23:59:53,818 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T23:59:53,818 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T23:59:53,818 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:53,818 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:53,818 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:59:53,818 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-09T23:59:53,818 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=387043539, stopped=false 2024-12-09T23:59:53,819 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=2907d75fbb3e,45191,1733788599119 2024-12-09T23:59:53,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45191-0x1000d103ede0000, quorum=127.0.0.1:60257, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T23:59:53,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37627-0x1000d103ede0001, quorum=127.0.0.1:60257, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T23:59:53,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45191-0x1000d103ede0000, quorum=127.0.0.1:60257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T23:59:53,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37627-0x1000d103ede0001, quorum=127.0.0.1:60257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T23:59:53,883 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T23:59:53,885 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45191-0x1000d103ede0000, quorum=127.0.0.1:60257, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T23:59:53,886 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T23:59:53,886 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37627-0x1000d103ede0001, quorum=127.0.0.1:60257, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T23:59:53,886 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T23:59:53,886 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:53,887 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2907d75fbb3e,37627,1733788599837' ***** 2024-12-09T23:59:53,887 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-09T23:59:53,888 INFO [RS:0;2907d75fbb3e:37627 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T23:59:53,888 INFO [RS:0;2907d75fbb3e:37627 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T23:59:53,888 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T23:59:53,889 INFO [RS:0;2907d75fbb3e:37627 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T23:59:53,889 INFO [RS:0;2907d75fbb3e:37627 {}] regionserver.HRegionServer(959): stopping server 2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:53,889 INFO [RS:0;2907d75fbb3e:37627 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T23:59:53,889 INFO [RS:0;2907d75fbb3e:37627 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;2907d75fbb3e:37627. 2024-12-09T23:59:53,889 DEBUG [RS:0;2907d75fbb3e:37627 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T23:59:53,889 DEBUG [RS:0;2907d75fbb3e:37627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:53,889 INFO [RS:0;2907d75fbb3e:37627 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T23:59:53,890 INFO [RS:0;2907d75fbb3e:37627 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T23:59:53,890 INFO [RS:0;2907d75fbb3e:37627 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T23:59:53,890 INFO [RS:0;2907d75fbb3e:37627 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-09T23:59:53,891 INFO [RS:0;2907d75fbb3e:37627 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-09T23:59:53,891 DEBUG [RS:0;2907d75fbb3e:37627 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-09T23:59:53,891 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T23:59:53,891 DEBUG [RS:0;2907d75fbb3e:37627 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T23:59:53,891 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T23:59:53,891 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T23:59:53,892 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T23:59:53,892 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T23:59:53,892 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-09T23:59:53,910 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/hbase/meta/1588230740/.tmp/info/4bf5753219b64ca59e15d95c60a65c21 is 95, key is TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203./info:/1733788636877/DeleteFamily/seqid=0 2024-12-09T23:59:53,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742382_1558 (size=6514) 2024-12-09T23:59:53,936 INFO [regionserver/2907d75fbb3e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T23:59:54,092 DEBUG [RS:0;2907d75fbb3e:37627 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T23:59:54,292 DEBUG [RS:0;2907d75fbb3e:37627 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T23:59:54,315 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/hbase/meta/1588230740/.tmp/info/4bf5753219b64ca59e15d95c60a65c21 2024-12-09T23:59:54,338 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/hbase/meta/1588230740/.tmp/ns/ab81f5e24f4242288a8867bc316c947b is 93, key is TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203./ns:/1733788636836/DeleteFamily/seqid=0 2024-12-09T23:59:54,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742383_1559 (size=5906) 2024-12-09T23:59:54,492 DEBUG [RS:0;2907d75fbb3e:37627 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T23:59:54,693 DEBUG [RS:0;2907d75fbb3e:37627 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T23:59:54,742 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/hbase/meta/1588230740/.tmp/ns/ab81f5e24f4242288a8867bc316c947b 2024-12-09T23:59:54,765 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/hbase/meta/1588230740/.tmp/rep_barrier/30383dcb8b774962bba5f838d725f04f is 102, key is TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203./rep_barrier:/1733788636836/DeleteFamily/seqid=0 2024-12-09T23:59:54,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742384_1560 (size=6025) 2024-12-09T23:59:54,834 INFO [regionserver/2907d75fbb3e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-09T23:59:54,834 INFO [regionserver/2907d75fbb3e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-09T23:59:54,893 DEBUG [RS:0;2907d75fbb3e:37627 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T23:59:55,094 DEBUG [RS:0;2907d75fbb3e:37627 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T23:59:55,169 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/hbase/meta/1588230740/.tmp/rep_barrier/30383dcb8b774962bba5f838d725f04f 2024-12-09T23:59:55,196 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/hbase/meta/1588230740/.tmp/table/e21e22cd75944487a4aaf5ca598898ae is 96, key is TestAcidGuarantees,,1733788604431.c1cc0773bc61ee89c3c2791bfea1e203./table:/1733788636836/DeleteFamily/seqid=0 2024-12-09T23:59:55,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742385_1561 (size=5936) 2024-12-09T23:59:55,294 DEBUG [RS:0;2907d75fbb3e:37627 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T23:59:55,494 DEBUG [RS:0;2907d75fbb3e:37627 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T23:59:55,601 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/hbase/meta/1588230740/.tmp/table/e21e22cd75944487a4aaf5ca598898ae 2024-12-09T23:59:55,611 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/hbase/meta/1588230740/.tmp/info/4bf5753219b64ca59e15d95c60a65c21 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/hbase/meta/1588230740/info/4bf5753219b64ca59e15d95c60a65c21 2024-12-09T23:59:55,617 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/hbase/meta/1588230740/info/4bf5753219b64ca59e15d95c60a65c21, entries=12, sequenceid=90, filesize=6.4 K 2024-12-09T23:59:55,619 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/hbase/meta/1588230740/.tmp/ns/ab81f5e24f4242288a8867bc316c947b as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/hbase/meta/1588230740/ns/ab81f5e24f4242288a8867bc316c947b 2024-12-09T23:59:55,625 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/hbase/meta/1588230740/ns/ab81f5e24f4242288a8867bc316c947b, entries=8, sequenceid=90, filesize=5.8 K 2024-12-09T23:59:55,625 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/hbase/meta/1588230740/.tmp/rep_barrier/30383dcb8b774962bba5f838d725f04f as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/hbase/meta/1588230740/rep_barrier/30383dcb8b774962bba5f838d725f04f 2024-12-09T23:59:55,629 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/hbase/meta/1588230740/rep_barrier/30383dcb8b774962bba5f838d725f04f, entries=6, sequenceid=90, filesize=5.9 K 2024-12-09T23:59:55,630 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/hbase/meta/1588230740/.tmp/table/e21e22cd75944487a4aaf5ca598898ae as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/hbase/meta/1588230740/table/e21e22cd75944487a4aaf5ca598898ae 2024-12-09T23:59:55,633 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/hbase/meta/1588230740/table/e21e22cd75944487a4aaf5ca598898ae, entries=7, sequenceid=90, filesize=5.8 K 2024-12-09T23:59:55,633 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1741ms, sequenceid=90, compaction requested=false 2024-12-09T23:59:55,637 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-09T23:59:55,638 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T23:59:55,638 INFO [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T23:59:55,638 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733788793891Running coprocessor pre-close hooks at 1733788793891Disabling compacts and flushes for region at 1733788793891Disabling writes for close at 1733788793892 (+1 ms)Obtaining lock to block concurrent updates at 1733788793892Preparing flush snapshotting stores in 1588230740 at 1733788793892Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733788793893 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733788793893Flushing 1588230740/info: creating writer at 1733788793894 (+1 ms)Flushing 1588230740/info: appending metadata at 1733788793910 (+16 ms)Flushing 1588230740/info: closing flushed file at 1733788793910Flushing 1588230740/ns: creating writer at 1733788794323 (+413 ms)Flushing 1588230740/ns: appending metadata at 1733788794337 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1733788794337Flushing 1588230740/rep_barrier: creating writer at 1733788794748 (+411 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733788794765 (+17 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733788794765Flushing 1588230740/table: creating writer at 1733788795178 (+413 ms)Flushing 1588230740/table: appending metadata at 1733788795196 (+18 ms)Flushing 1588230740/table: closing flushed file at 1733788795196Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@233ca480: reopening flushed file at 1733788795610 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d540b0b: reopening flushed file at 1733788795618 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@207b612: reopening flushed file at 1733788795625 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b89ec3b: reopening flushed file at 1733788795629 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1741ms, sequenceid=90, compaction requested=false at 1733788795633 (+4 ms)Writing region close event to WAL at 1733788795635 (+2 ms)Running coprocessor post-close hooks at 1733788795637 (+2 ms)Closed at 1733788795638 (+1 ms) 2024-12-09T23:59:55,638 DEBUG [RS_CLOSE_META-regionserver/2907d75fbb3e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-09T23:59:55,695 INFO [RS:0;2907d75fbb3e:37627 {}] regionserver.HRegionServer(976): stopping server 2907d75fbb3e,37627,1733788599837; all regions closed. 2024-12-09T23:59:55,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741834_1010 (size=25344) 2024-12-09T23:59:55,704 DEBUG [RS:0;2907d75fbb3e:37627 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/oldWALs 2024-12-09T23:59:55,704 INFO [RS:0;2907d75fbb3e:37627 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2907d75fbb3e%2C37627%2C1733788599837.meta:.meta(num 1733788603974) 2024-12-09T23:59:55,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741832_1008 (size=22681250) 2024-12-09T23:59:55,708 DEBUG [RS:0;2907d75fbb3e:37627 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/oldWALs 2024-12-09T23:59:55,708 INFO [RS:0;2907d75fbb3e:37627 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2907d75fbb3e%2C37627%2C1733788599837:(num 1733788602988) 2024-12-09T23:59:55,708 DEBUG [RS:0;2907d75fbb3e:37627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:59:55,708 INFO [RS:0;2907d75fbb3e:37627 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T23:59:55,708 INFO [RS:0;2907d75fbb3e:37627 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T23:59:55,709 INFO [RS:0;2907d75fbb3e:37627 {}] hbase.ChoreService(370): Chore service for: regionserver/2907d75fbb3e:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-09T23:59:55,709 INFO [RS:0;2907d75fbb3e:37627 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T23:59:55,709 INFO [regionserver/2907d75fbb3e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T23:59:55,710 INFO [RS:0;2907d75fbb3e:37627 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37627 2024-12-09T23:59:55,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37627-0x1000d103ede0001, quorum=127.0.0.1:60257, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2907d75fbb3e,37627,1733788599837 2024-12-09T23:59:55,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45191-0x1000d103ede0000, quorum=127.0.0.1:60257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T23:59:55,775 INFO [RS:0;2907d75fbb3e:37627 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T23:59:55,777 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2907d75fbb3e,37627,1733788599837] 2024-12-09T23:59:55,793 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2907d75fbb3e,37627,1733788599837 already deleted, retry=false 2024-12-09T23:59:55,793 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2907d75fbb3e,37627,1733788599837 expired; onlineServers=0 2024-12-09T23:59:55,794 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '2907d75fbb3e,45191,1733788599119' ***** 2024-12-09T23:59:55,794 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-09T23:59:55,794 INFO [M:0;2907d75fbb3e:45191 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T23:59:55,794 INFO [M:0;2907d75fbb3e:45191 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T23:59:55,794 DEBUG [M:0;2907d75fbb3e:45191 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-09T23:59:55,795 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-09T23:59:55,795 DEBUG [M:0;2907d75fbb3e:45191 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-09T23:59:55,795 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster-HFileCleaner.large.0-1733788602657 {}] cleaner.HFileCleaner(306): Exit Thread[master/2907d75fbb3e:0:becomeActiveMaster-HFileCleaner.large.0-1733788602657,5,FailOnTimeoutGroup] 2024-12-09T23:59:55,795 DEBUG [master/2907d75fbb3e:0:becomeActiveMaster-HFileCleaner.small.0-1733788602658 {}] cleaner.HFileCleaner(306): Exit Thread[master/2907d75fbb3e:0:becomeActiveMaster-HFileCleaner.small.0-1733788602658,5,FailOnTimeoutGroup] 2024-12-09T23:59:55,795 INFO [M:0;2907d75fbb3e:45191 {}] hbase.ChoreService(370): Chore service for: master/2907d75fbb3e:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-09T23:59:55,795 INFO [M:0;2907d75fbb3e:45191 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T23:59:55,796 DEBUG [M:0;2907d75fbb3e:45191 {}] master.HMaster(1795): Stopping service threads 2024-12-09T23:59:55,796 INFO [M:0;2907d75fbb3e:45191 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-09T23:59:55,796 INFO [M:0;2907d75fbb3e:45191 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T23:59:55,796 ERROR [M:0;2907d75fbb3e:45191 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:35525 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:35525,5,PEWorkerGroup] 2024-12-09T23:59:55,798 INFO [M:0;2907d75fbb3e:45191 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T23:59:55,798 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-09T23:59:55,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45191-0x1000d103ede0000, quorum=127.0.0.1:60257, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-09T23:59:55,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45191-0x1000d103ede0000, quorum=127.0.0.1:60257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T23:59:55,801 DEBUG [M:0;2907d75fbb3e:45191 {}] zookeeper.ZKUtil(347): master:45191-0x1000d103ede0000, quorum=127.0.0.1:60257, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-09T23:59:55,801 WARN [M:0;2907d75fbb3e:45191 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-09T23:59:55,802 INFO [M:0;2907d75fbb3e:45191 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/.lastflushedseqids 2024-12-09T23:59:55,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742386_1562 (size=72) 2024-12-09T23:59:55,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37627-0x1000d103ede0001, quorum=127.0.0.1:60257, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T23:59:55,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37627-0x1000d103ede0001, quorum=127.0.0.1:60257, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T23:59:55,887 INFO [RS:0;2907d75fbb3e:37627 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T23:59:55,887 INFO [RS:0;2907d75fbb3e:37627 {}] regionserver.HRegionServer(1031): Exiting; stopping=2907d75fbb3e,37627,1733788599837; zookeeper connection closed. 2024-12-09T23:59:55,888 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2629948a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2629948a 2024-12-09T23:59:55,889 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-09T23:59:56,212 INFO [M:0;2907d75fbb3e:45191 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T23:59:56,213 INFO [M:0;2907d75fbb3e:45191 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-09T23:59:56,213 DEBUG [M:0;2907d75fbb3e:45191 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T23:59:56,213 INFO [M:0;2907d75fbb3e:45191 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T23:59:56,213 DEBUG [M:0;2907d75fbb3e:45191 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T23:59:56,214 DEBUG [M:0;2907d75fbb3e:45191 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T23:59:56,214 DEBUG [M:0;2907d75fbb3e:45191 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T23:59:56,214 INFO [M:0;2907d75fbb3e:45191 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=763.12 KB heapSize=933.79 KB 2024-12-09T23:59:56,256 DEBUG [M:0;2907d75fbb3e:45191 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c9d5050d08384895916847b931930c3d is 82, key is hbase:meta,,1/info:regioninfo/1733788604044/Put/seqid=0 2024-12-09T23:59:56,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742387_1563 (size=5672) 2024-12-09T23:59:56,661 INFO [M:0;2907d75fbb3e:45191 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2101 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c9d5050d08384895916847b931930c3d 2024-12-09T23:59:56,687 DEBUG [M:0;2907d75fbb3e:45191 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1776cf62df77446fb5e1b55dd520caec is 2480, key is \x00\x00\x00\x00\x00\x00\x00!/proc:d/1733788640128/Put/seqid=0 2024-12-09T23:59:56,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742388_1564 (size=45770) 2024-12-09T23:59:57,092 INFO [M:0;2907d75fbb3e:45191 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=762.56 KB at sequenceid=2101 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1776cf62df77446fb5e1b55dd520caec 2024-12-09T23:59:57,099 INFO [M:0;2907d75fbb3e:45191 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1776cf62df77446fb5e1b55dd520caec 2024-12-09T23:59:57,116 DEBUG [M:0;2907d75fbb3e:45191 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/bdad67aa0dd04c58b254a058359a58f1 is 69, key is 2907d75fbb3e,37627,1733788599837/rs:state/1733788602737/Put/seqid=0 2024-12-09T23:59:57,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073742389_1565 (size=5156) 2024-12-09T23:59:57,520 INFO [M:0;2907d75fbb3e:45191 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2101 (bloomFilter=true), to=hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/bdad67aa0dd04c58b254a058359a58f1 2024-12-09T23:59:57,530 DEBUG [M:0;2907d75fbb3e:45191 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c9d5050d08384895916847b931930c3d as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c9d5050d08384895916847b931930c3d 2024-12-09T23:59:57,534 INFO [M:0;2907d75fbb3e:45191 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c9d5050d08384895916847b931930c3d, entries=8, sequenceid=2101, filesize=5.5 K 2024-12-09T23:59:57,534 DEBUG [M:0;2907d75fbb3e:45191 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1776cf62df77446fb5e1b55dd520caec as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1776cf62df77446fb5e1b55dd520caec 2024-12-09T23:59:57,537 INFO [M:0;2907d75fbb3e:45191 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1776cf62df77446fb5e1b55dd520caec 2024-12-09T23:59:57,537 INFO [M:0;2907d75fbb3e:45191 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1776cf62df77446fb5e1b55dd520caec, entries=172, sequenceid=2101, filesize=44.7 K 2024-12-09T23:59:57,538 DEBUG [M:0;2907d75fbb3e:45191 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/bdad67aa0dd04c58b254a058359a58f1 as hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/bdad67aa0dd04c58b254a058359a58f1 2024-12-09T23:59:57,541 INFO [M:0;2907d75fbb3e:45191 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35525/user/jenkins/test-data/706881ae-9be8-b71c-1448-ffd0243c2f2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/bdad67aa0dd04c58b254a058359a58f1, entries=1, sequenceid=2101, filesize=5.0 K 2024-12-09T23:59:57,542 INFO [M:0;2907d75fbb3e:45191 {}] regionserver.HRegion(3140): Finished flush of dataSize ~763.12 KB/781430, heapSize ~933.49 KB/955896, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1328ms, sequenceid=2101, compaction requested=false 2024-12-09T23:59:57,543 INFO [M:0;2907d75fbb3e:45191 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T23:59:57,543 DEBUG [M:0;2907d75fbb3e:45191 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733788796213Disabling compacts and flushes for region at 1733788796213Disabling writes for close at 1733788796214 (+1 ms)Obtaining lock to block concurrent updates at 1733788796214Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733788796214Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=781430, getHeapSize=956136, getOffHeapSize=0, getCellsCount=2298 at 1733788796215 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733788796234 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733788796234Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733788796256 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733788796256Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733788796671 (+415 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733788796686 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733788796686Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733788797099 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733788797115 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733788797115Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d9b9eed: reopening flushed file at 1733788797530 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f6bddec: reopening flushed file at 1733788797534 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@198f0d7c: reopening flushed file at 1733788797537 (+3 ms)Finished flush of dataSize ~763.12 KB/781430, heapSize ~933.49 KB/955896, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1328ms, sequenceid=2101, compaction requested=false at 1733788797542 (+5 ms)Writing region close event to WAL at 1733788797543 (+1 ms)Closed at 1733788797543 2024-12-09T23:59:57,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42911 is added to blk_1073741830_1006 (size=919202) 2024-12-09T23:59:57,546 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T23:59:57,546 INFO [M:0;2907d75fbb3e:45191 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-09T23:59:57,547 INFO [M:0;2907d75fbb3e:45191 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45191 2024-12-09T23:59:57,547 INFO [M:0;2907d75fbb3e:45191 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T23:59:57,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45191-0x1000d103ede0000, quorum=127.0.0.1:60257, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T23:59:57,710 INFO [M:0;2907d75fbb3e:45191 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T23:59:57,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45191-0x1000d103ede0000, quorum=127.0.0.1:60257, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T23:59:57,755 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T23:59:57,757 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T23:59:57,757 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T23:59:57,757 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T23:59:57,758 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8bc2ec3c-7335-1808-2d8f-cd0f163ec5b4/hadoop.log.dir/,STOPPED} 2024-12-09T23:59:57,760 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-09T23:59:57,760 WARN [BP-1787334953-172.17.0.2-1733788595887 heartbeating to localhost/127.0.0.1:35525 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-09T23:59:57,760 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-09T23:59:57,760 WARN [BP-1787334953-172.17.0.2-1733788595887 heartbeating to localhost/127.0.0.1:35525 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1787334953-172.17.0.2-1733788595887 (Datanode Uuid c0f3510f-a7b2-4204-a23a-5dac7990f2a7) service to localhost/127.0.0.1:35525 2024-12-09T23:59:57,763 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8bc2ec3c-7335-1808-2d8f-cd0f163ec5b4/cluster_96f75c57-c245-065f-99c2-7563c90f684d/data/data1/current/BP-1787334953-172.17.0.2-1733788595887 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T23:59:57,763 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8bc2ec3c-7335-1808-2d8f-cd0f163ec5b4/cluster_96f75c57-c245-065f-99c2-7563c90f684d/data/data2/current/BP-1787334953-172.17.0.2-1733788595887 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T23:59:57,764 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-09T23:59:57,771 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T23:59:57,772 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T23:59:57,772 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T23:59:57,772 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T23:59:57,772 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8bc2ec3c-7335-1808-2d8f-cd0f163ec5b4/hadoop.log.dir/,STOPPED} 2024-12-09T23:59:57,786 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-09T23:59:57,887 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down